Skip sqlite test for python 3.11 and higher (#194)

[versions-manifest] Update for release from 10/25/2022

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '30 8 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/get-python-versions.yml

@@ -12,6 +12,7 @@ defaults:
 
 jobs:
   find_new_versions:
+    if: github.repository_owner == 'actions'
     name: Find new versions
     runs-on: ubuntu-latest
     outputs:

.github/workflows/python-builder.yml

@@ -14,7 +14,7 @@ on:
       PLATFORMS:
         description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
         required: true
-        default: 'ubuntu-18.04,ubuntu-20.04,macos-10.15,windows-2019_x64,windows-2019_x86'
+        default: 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86'
   pull_request:
     paths-ignore:
     - 'versions-manifest.json'
@@ -38,7 +38,7 @@ jobs:
       - name: Generate execution matrix
         id: generate-matrix
         run: |
-          $configurations = "${{ github.event.inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,macos-10.15,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
+          $configurations = "${{ github.event.inputs.platforms || 'ubuntu-18.04,ubuntu-20.04,ubuntu-22.04,macos-10.15,windows-2019_x64,windows-2019_x86' }}".Split(",").Trim()
           $matrix = @()
 
           foreach ($configuration in $configurations) {
@@ -131,7 +131,7 @@ jobs:
         working-directory: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
 
       - name: Setup Python ${{ env.VERSION }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ env.VERSION }}
           architecture: ${{ matrix.arch }}

.github/workflows/releases-validation.yml

@@ -18,7 +18,7 @@ jobs:
         python: [3.5.4, 3.6.7, 3.7.5, 3.8.1]
     steps:
     - name: setup-python ${{ matrix.python }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python }}
 

.github/workflows/validate-manifest.yml

@@ -19,6 +19,7 @@ defaults:
 
 jobs:
   validation:
+    if: github.repository_owner == 'actions'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

builders/macos-python-builder.psm1

@@ -7,6 +7,10 @@ class macOSPythonBuilder : NixPythonBuilder {
 
     .DESCRIPTION
     Contains methods that required to build macOS Python artifact from sources. Inherited from base NixPythonBuilder.
+    
+    While python.org provides precompiled binaries for macOS, switching to them risks breaking existing customers.
+    If we wanted to start using the official binaries instead of building from source, we should avoid changing previous versions
+    so we remain backwards compatible.
 
     .PARAMETER platform
     The full name of platform for which Python should be built.
@@ -42,6 +46,15 @@ class macOSPythonBuilder : NixPythonBuilder {
         $configureString += " --enable-shared"
         $configureString += " --with-lto"
 
+        ### For Python versions which support it, compile a universal2 (arm64 + x86_64 hybrid) build. The arm64 slice
+        ### will never be used itself by a Github Actions runner but using a universal2 Python is the only way to build
+        ### universal2 C extensions and wheels. This is supported by Python >= 3.10 and was backported to Python >=
+        ### 3.9.1 and >= 3.8.10.
+        ### Disabled, discussion: https://github.com/actions/python-versions/pull/114
+        # if ($this.Version -ge "3.8.10" -and $this.Version -ne "3.8.13" -and $this.Version -ne "3.9.0" ) {
+        #     $configureString += " --enable-universalsdk --with-universal-archs=universal2"
+        # }
+
         ### OS X 10.11, Apple no longer provides header files for the deprecated system version of OpenSSL.
         ### Solution is to install these libraries from a third-party package manager,
         ### and then add the appropriate paths for the header and library files to configure command.
@@ -51,6 +64,9 @@ class macOSPythonBuilder : NixPythonBuilder {
             $env:CFLAGS = "-I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/zlib/include"
         } else {
             $configureString += " --with-openssl=/usr/local/opt/openssl@1.1"
+            if ($this.Version -gt "3.7.12") {
+                $configureString += " --with-tcltk-includes='-I /usr/local/opt/tcl-tk/include' --with-tcltk-libs='-L/usr/local/opt/tcl-tk/lib -ltcl8.6 -ltk8.6'"
+	    }
         }
 
         ### Compile with support of loadable sqlite extensions. Unavailable for Python 2.*
@@ -62,6 +78,97 @@ class macOSPythonBuilder : NixPythonBuilder {
             $env:CPPFLAGS += "-I$(brew --prefix sqlite3)/include"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
+
+    [string] GetPkgName() {
+        <#
+        .SYNOPSIS
+        Return Python installation Package.
+        #>
+
+        $nativeVersion = Convert-Version -version $this.Version
+        $architecture = "-macos11"
+        $extension = ".pkg"
+
+        $pkg = "python-${nativeVersion}${architecture}${extension}"
+
+        return $pkg
+    }
+
+    [uri] GetPkgUri() {
+        <#
+        .SYNOPSIS
+        Get base Python URI and return complete URI for Python installation package.
+        #>
+
+        $base = $this.GetBaseUri()
+        $versionName = $this.GetBaseVersion()
+        $pkg = $this.GetPkgName()
+
+        $uri = "${base}/${versionName}/${pkg}"
+
+        return $uri
+    }
+
+    [string] DownloadPkg() {
+        <#
+        .SYNOPSIS
+        Download Python installation executable into artifact location.
+        #>
+
+        $pkgUri = $this.GetPkgUri()
+
+        Write-Host "Sources URI: $pkgUri"
+        $pkgLocation = Download-File -Uri $pkgUri -OutputFolder $this.WorkFolderLocation
+        Write-Debug "Done; Package location: $pkgLocation"
+
+        New-Item -Path $this.WorkFolderLocation -Name "build_output.txt"  -ItemType File
+        return $pkgLocation
+    }
+
+    [void] CreateInstallationScriptPkg() {
+        <#
+        .SYNOPSIS
+        Create Python artifact installation script based on specified template.
+        #>
+
+        $installationTemplateLocation = Join-Path -Path $this.InstallationTemplatesLocation -ChildPath "macos-pkg-setup-template.sh"
+        $installationTemplateContent = Get-Content -Path $installationTemplateLocation -Raw
+        $installationScriptLocation = New-Item -Path $this.WorkFolderLocation -Name $this.InstallationScriptName  -ItemType File
+
+        $variablesToReplace = @{
+            "{{__VERSION_FULL__}}" = $this.Version;
+            "{{__PKG_NAME__}}" = $this.GetPkgName();
+        }
+
+        $variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }
+        $installationTemplateContent | Out-File -FilePath $installationScriptLocation
+        Write-Debug "Done; Installation script location: $installationScriptLocation)"
+    }
+
+    [void] Build() {
+        <#
+        .SYNOPSIS
+        Generates Python artifact from downloaded Python installation executable.
+        #>
+
+        $PkgVersion = [semver]"3.11.0-beta.1"
+
+        if ($this.Version -ge $PkgVersion) {
+            Write-Host "Download Python $($this.Version) [$($this.Architecture)] package..."
+            $this.DownloadPkg()
+
+            Write-Host "Create installation script..."
+            $this.CreateInstallationScriptPkg()
+        } else {
+            ([NixPythonBuilder]$this).Build()
+        }
+
+        Write-Host "Archive artifact"
+        $this.ArchiveArtifact()
+    }
 }

builders/ubuntu-python-builder.psm1

@@ -48,6 +48,9 @@ class UbuntuPythonBuilder : NixPythonBuilder {
             $configureString += " --enable-loadable-sqlite-extensions"
         }
 
+        Write-Host "The passed configure options are: "
+        Write-Host $configureString
+
         Execute-Command -Command $configureString
     }
 

installers/macos-pkg-setup-template.sh

@@ -0,0 +1,72 @@
+set -e
+
+PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
+PYTHON_PKG_NAME="{{__PKG_NAME__}}"
+MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
+MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
+
+PYTHON_MAJOR=python$MAJOR_VERSION
+PYTHON_MAJOR_DOT_MINOR=python$MAJOR_VERSION.$MINOR_VERSION
+PYTHON_MAJOR_MINOR=python$MAJOR_VERSION$MINOR_VERSION
+
+if [ -z ${AGENT_TOOLSDIRECTORY+x} ]; then
+    # No AGENT_TOOLSDIRECTORY on GitHub images
+    TOOLCACHE_ROOT=$RUNNER_TOOL_CACHE
+else
+    TOOLCACHE_ROOT=$AGENT_TOOLSDIRECTORY
+fi
+
+PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
+PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
+PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/x64
+PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
+PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
+
+echo "Check if Python hostedtoolcache folder exist..."
+if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
+    echo "Creating Python hostedtoolcache folder..."
+    mkdir -p $PYTHON_TOOLCACHE_PATH
+else
+    # remove ALL other directories for same major.minor python versions
+    find $PYTHON_TOOLCACHE_PATH -name "${MAJOR_VERSION}.${MINOR_VERSION}.*"|while read python_version;do
+        python_version_x64="$python_version/x64"
+        if [ -e "$python_version_x64" ];then
+            echo "Deleting Python $python_version_x64"
+            rm -rf "$python_version_x64"
+        fi
+    done
+fi
+
+echo "Install Python binaries from prebuilt package"
+sudo installer -pkg $PYTHON_PKG_NAME -target /
+
+echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
+echo "Create Python $PYTHON_FULL_VERSION folder"
+mkdir -p $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+cd $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
+
+ln -s "${PYTHON_FRAMEWORK_PATH}/bin" bin
+ln -s "${PYTHON_FRAMEWORK_PATH}/include" include
+ln -s "${PYTHON_FRAMEWORK_PATH}/share" share
+ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
+
+echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
+ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
+
+cd bin/
+ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
+if [ ! -f python ]; then
+    ln -s $PYTHON_MAJOR_DOT_MINOR python
+fi
+
+chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
+
+echo "Upgrading pip..."
+./python -m ensurepip
+./python -m pip install --ignore-installed pip --disable-pip-version-check --no-warn-script-location
+
+echo "Install OpenSSL certificates"
+sh -e "${PYTHON_APPLICATION_PATH}/Install Certificates.command"
+
+echo "Create complete file"
+touch $PYTHON_TOOLCACHE_VERSION_PATH/x64.complete

tests/python-tests.ps1

@@ -49,7 +49,14 @@ Describe "Tests" {
         "python ./sources/simple-test.py" | Should -ReturnZeroExitCode
     }
 
-    if (($Version -ge "3.2.0") -and -not ([semver]"$($Version.Major).$($Version.Minor)" -eq [semver]"3.11" -and $Version.PreReleaseLabel)) {
+    # linux has no display name and no $DISPLAY environment variable - skip tk test
+    # if (-not (($Platform -match "ubuntu") -or ($Platform -match "linux"))) {
+    #     It "Check if tcl/tk has the same headed and library versions" {
+	#     "python ./sources/tcltk.py" | Should -ReturnZeroExitCode
+    #     }
+    # }
+
+    if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0")) {
         It "Check if sqlite3 module is installed" {
             "python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
         }
@@ -89,4 +96,8 @@ Describe "Tests" {
             "./dist/simple-test" | Should -ReturnZeroExitCode
         }
     }
+
+    It "Check urlopen with HTTPS works" {
+        "python ./sources/python-urlopen-https.py" | Should -ReturnZeroExitCode
+    }
 }

tests/sources/python-config-test.py

@@ -10,6 +10,12 @@ os_type = platform.system()
 version = sys.argv[1]
 nativeVersion = sys.argv[2]
 
+versions=version.split(".")
+version_major=int(versions[0])
+version_minor=int(versions[1])
+
+pkg_installer = os_type == 'Darwin' and (version_major == 3 and version_minor >= 11)
+
 lib_dir_path = sysconfig.get_config_var('LIBDIR')
 ld_library_name = sysconfig.get_config_var('LDLIBRARY')
 
@@ -19,7 +25,11 @@ have_libreadline = sysconfig.get_config_var("HAVE_LIBREADLINE")
 ### Define expected variables
 if os_type == 'Linux': expected_ld_library_extension = 'so'
 if os_type == 'Darwin': expected_ld_library_extension = 'dylib'
-expected_lib_dir_path = '{0}/Python/{1}/x64/lib'.format(os.getenv("AGENT_TOOLSDIRECTORY"), version)
+
+if pkg_installer:
+    expected_lib_dir_path = f'/Library/Frameworks/Python.framework/Versions/{version_major}.{version_minor}/lib'
+else:
+    expected_lib_dir_path = f'{os.getenv("AGENT_TOOLSDIRECTORY")}/Python/{version}/x64/lib'
 
 # Check modules
 ### Validate libraries path
@@ -38,7 +48,8 @@ if is_shared:
         exit(1)
 else:
     print('%s was built without shared extensions' % ld_library_name)
-    exit(1)
+    if not pkg_installer:
+        exit(1)
 
 ### Validate macOS
 if os_type == 'Darwin':
@@ -59,12 +70,14 @@ if os_type == 'Darwin':
 
         if openssl_includes != expected_openssl_includes:
             print('Invalid openssl_includes: %s; Expected: %s' % (openssl_includes, expected_openssl_includes))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
         if openssl_ldflags != expected_openssl_ldflags:
             print('Invalid openssl_ldflags: %s; Expected: %s' % (openssl_ldflags, expected_openssl_ldflags))
-            exit(1)
+            if not pkg_installer:
+                exit(1)
 
 ### Validate libreadline
 if not have_libreadline:
     print('Missing libreadline')
-    exit(1)
+    exit(1)

tests/sources/python-modules.py

@@ -7,6 +7,7 @@ from __future__ import print_function
 
 import importlib
 import sys
+import platform
 
 # The Python standard library as of Python 3.0
 standard_library = [
@@ -265,6 +266,11 @@ if sys.version_info >= (3, 10):
 if sys.version_info >= (3, 11):
     standard_library.remove('binhex')
 
+# Exclude tkinter and turtle for Python 3.11 alpha temporarily
+if sys.version_info >= (3, 11) and platform.system() == 'Linux' and '18.04' in platform.version():
+    standard_library.remove('tkinter')
+    standard_library.remove('turtle')
+
 # Remove tkinter and Easter eggs
 excluded_modules = [
     'antigravity',

tests/sources/python-urlopen-https.py

@@ -0,0 +1,10 @@
+import sys
+
+if sys.version_info[0] == 2:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+response = urlopen("https://raw.githubusercontent.com/actions/python-versions/c641695f6a07526c18f10e374e503e649fef9427/.gitignore")
+data = response.read()
+assert len(data) == 140, len(data)

tests/sources/tcltk.py

@@ -0,0 +1,10 @@
+import tkinter
+import _tkinter
+
+header = _tkinter.TK_VERSION
+lib = tkinter.Tk().getvar('tk_version')
+
+if lib != header:
+  print('header version=' + header)
+  print('lib version=' + lib)
+  exit(1)