Merge branch 'main' into releases/ubuntu20/20200908-docs

images/linux/scripts/base/apt.sh

@@ -11,8 +11,10 @@ systemctl disable apt-daily-upgrade.service
 # Configure apt to always assume Y
 echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/90assumeyes
 
-# Use apt-fast for parallel downloads
+# Install aria2 and jq
 apt-get install aria2
+apt-get install jq
+# Use apt-fast for parallel downloads
 add-apt-repository -y ppa:apt-fast/stable
 apt-get update
-apt-get -y install apt-fast
+apt-get install apt-fast

images/linux/scripts/helpers/containercache.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-################################################################################
-##  File:  containercache.sh
-##  Desc:  Prepulls Docker images used in build tasks and templates
-################################################################################
-
-source $HELPER_SCRIPTS/document.sh
-
-# Check prereqs
-echo "Checking prereqs for image pulls"
-if ! command -v docker; then
-    echo "Docker is not installed, cant pull images"
-    exit 1
-fi
-
-# Information output
-systemctl status docker --no-pager
-
-# Pull images
-images=(
-    docker.io/jekyll/builder
-    mcr.microsoft.com/azure-pipelines/node8-typescript
-)
-
-for image in "${images[@]}"; do
-    docker pull "$image"
-done
-
-## Add container information to the metadata file
- DocumentInstalledItem "Cached container images"
-
-while read -r line; do
-    DocumentInstalledItemIndent "$line"
-done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')"

images/linux/scripts/installers/Install-Toolset.ps1

@@ -27,10 +27,10 @@ Function Install-Asset {
 $ErrorActionPreference = "Stop"
 
 # Get toolset content
-$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
+$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
 $toolsToInstall = @("Python", "Node", "Boost", "Go")
 
-$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache | Where-Object {$ToolsToInstall -contains $_.Name}
+$tools = ConvertFrom-Json -InputObject $toolset | Select-Object -ExpandProperty toolcache | Where-Object {$ToolsToInstall -contains $_.Name}
 
 foreach ($tool in $tools) {
     # Get versions manifest for current tool

images/linux/scripts/installers/Validate-Toolset.ps1

@@ -46,8 +46,8 @@ $toolsExecutables = @{
 }
 
 # Get toolset content
-$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
+$toolset = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw
-$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache
+$tools = ConvertFrom-Json -InputObject $toolset | Select-Object -ExpandProperty toolcache
 
 foreach($tool in $tools) {
     Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItem '$($tool.name):'`""

images/linux/scripts/installers/android.sh

@@ -42,12 +42,12 @@ else
     exit 1
 fi
 
-toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
-platforms=$(cat $toolsetJson  | jq -r '.android.platform_list[]|"platforms;" + .')
+platforms=$(jq -r '.android.platform_list[]|"platforms;" + .' $toolset)
-buildtools=$(cat $toolsetJson  | jq -r '.android.build_tools[]|"build-tools;" + .')
+buildtools=$(jq -r '.android.build_tools[]|"build-tools;" + .' $toolset)
-extras=$(cat $toolsetJson  | jq -r '.android.extra_list[]|"extras;" + .')
+extras=$(jq -r '.android.extra_list[]|"extras;" + .' $toolset)
-addons=$(cat $toolsetJson  | jq -r '.android.addon_list[]|"add-ons;" + .')
+addons=$(jq -r '.android.addon_list[]|"add-ons;" + .' $toolset)
-additional=$(cat $toolsetJson  | jq -r '.android.additional_tools[]')
+additional=$(jq -r '.android.additional_tools[]' $toolset)
 
 # Install the following SDKs and build tools, passing in "y" to accept licenses.
 echo "y" | ${ANDROID_SDK_ROOT}/tools/bin/sdkmanager $platforms $buildtools $extras $google_api_list $addons $additional

images/linux/scripts/installers/azpowershell.sh

@@ -12,8 +12,8 @@ source $HELPER_SCRIPTS/os.sh
 if isUbuntu20 ; then
     versions=$(pwsh -Command '(Find-Module -Name Az).Version')
 else
-    toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+    toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
-    versions=$(cat $toolsetJson | jq -r '.azureModules[] | select(.name | contains("az")) | .versions[]')
+    versions=$(jq -r '.azureModules[] | select(.name | contains("az")) | .versions[]' $toolset)
 fi
 
 # Install Azure CLI (instructions taken from https://docs.microsoft.com/en-us/cli/azure/install-azure-cli)

images/linux/scripts/installers/basic.sh

@@ -4,90 +4,13 @@
 ##  Desc:  Installs basic command line utilities and dev packages
 ################################################################################
 
+set -e
 # Source the helpers for use with the script
 source $HELPER_SCRIPTS/document.sh
-source $HELPER_SCRIPTS/os.sh
-
-set -e
-
-common_packages="dnsutils
-                 iproute2
-                 iputils-ping
-                 libunwind8
-                 locales
-                 openssh-client
-                 tzdata
-                 upx
-                 zstd
-                 libxkbfile-dev
-                 pkg-config
-                 libsecret-1-dev
-                 libxss1
-                 libgconf-2-4
-                 dbus
-                 xvfb
-                 libgbm-dev
-                 libgtk-3-0
-                 tk
-                 fakeroot
-                 dpkg
-                 rpm
-                 xz-utils
-                 xorriso
-                 zsync
-                 gnupg2
-                 lib32z1
-                 texinfo
-                 libsqlite3-dev
-                 libc++-dev
-                 libc++abi-dev"
-
-cmd_packages="curl
-              file
-              ftp
-              jq
-              netcat
-              ssh
-              parallel
-              rsync
-              shellcheck
-              sudo
-              telnet
-              time
-              unzip
-              zip
-              wget
-              m4
-              bison
-              flex
-              patchelf
-              bzip2
-              sqlite3
-              brotli
-              yamllint"
-
-if isUbuntu20 ; then
-    echo "Install python2"
-    apt-get install -y --no-install-recommends python-is-python2
-fi
-
-echo "Install libcurl"
-if isUbuntu16 || isUbuntu18; then
-   libcurelVer="libcurl3"
-fi
-
-if isUbuntu20 ; then
-    libcurelVer="libcurl4"
-fi
-
-apt-get install -y --no-install-recommends $libcurelVer
-
-# install additional packages only for Ubuntu16.04
-if isUbuntu16; then
-    common_packages="$common_packages
-            libicu55"
-fi
 
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+common_packages=$(jq -r ".apt.common_packages[]" $toolset)
+cmd_packages=$(jq -r ".apt.cmd_packages[]" $toolset)
 for package in $common_packages $cmd_packages; do
     echo "Install $package"
     apt-get install -y --no-install-recommends $package
@@ -108,5 +31,3 @@ DocumentInstalledItem "Basic packages:"
 for package in $common_packages $cmd_packages; do
     DocumentInstalledItemIndent $package
 done
-
-DocumentInstalledItemIndent "$libcurelVer"

images/linux/scripts/installers/docker-moby.sh

@@ -3,19 +3,20 @@
 ##  File:  docker-moby.sh
 ##  Desc:  Installs docker onto the image
 ################################################################################
+set -e
 
+# Source the helpers for use with the script
 source $HELPER_SCRIPTS/document.sh
 source $HELPER_SCRIPTS/install.sh
 source $HELPER_SCRIPTS/os.sh
 
-docker_package=moby
-
 # There is no stable docker-moby for Ubuntu 20 at the moment
 if isUbuntu20 ; then
     add-apt-repository "deb [arch=amd64,armhf,arm64] https://packages.microsoft.com/ubuntu/20.04/prod testing main"
 fi
 
-## Check to see if docker is already installed
+# Check to see if docker is already installed
+docker_package=moby
 echo "Determing if Docker ($docker_package) is installed"
 if ! IsPackageInstalled $docker_package; then
     echo "Docker ($docker_package) was not found. Installing..."
@@ -44,26 +45,17 @@ else
     echo "Docker-moby and Docker-buildx checking the successfull"
     # Docker daemon takes time to come up after installing
     sleep 10
-    set -e
     docker info
-    set +e
 fi
 
-docker pull node:10
+# Pull images
-docker pull node:12
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
-docker pull buildpack-deps:stretch
+images=$(jq -r '.docker.images[]' $toolset)
-docker pull buildpack-deps:buster
+for image in $images; do
-docker pull node:10-alpine
+    docker pull "$image"
-docker pull node:12-alpine
+done
-docker pull debian:8
-docker pull debian:9
-docker pull alpine:3.7
-docker pull alpine:3.8
-docker pull alpine:3.9
-docker pull alpine:3.10
-docker pull ubuntu:14.04
 
-## Add version information to the metadata file
+# Add version information to the metadata file
 echo "Documenting Docker version"
 docker_version=$(docker -v)
 DocumentInstalledItem "Docker-Moby ($docker_version)"
@@ -71,3 +63,9 @@ DocumentInstalledItem "Docker-Moby ($docker_version)"
 echo "Documenting Docker-buildx version"
 DOCKER_BUILDX_VERSION=$(docker buildx version | cut -d ' ' -f2)
 DocumentInstalledItem "Docker-Buildx ($DOCKER_BUILDX_VERSION)"
+
+# Add container information to the metadata file
+DocumentInstalledItem "Cached container images"
+while read -r line; do
+    DocumentInstalledItemIndent "$line"
+done <<< "$(docker images --digests --format '{{.Repository}}:{{.Tag}} (Digest: {{.Digest}})')"

images/linux/scripts/installers/pypy.sh

@@ -75,8 +75,8 @@ uri="https://downloads.python.org/pypy/"
 download_with_retries $uri "/tmp" "pypyUrls.html" compressed
 pypyVersions="$(cat /tmp/pypyUrls.html | grep 'linux64' | awk -v uri="$uri" -F'>|<' '{print uri$5}')"
 
-toolsetJson="$INSTALLER_SCRIPT_FOLDER/toolset.json"
+toolset="$INSTALLER_SCRIPT_FOLDER/toolset.json"
-toolsetVersions=$(cat $toolsetJson | jq -r '.toolcache[] | select(.name | contains("PyPy")) | .versions[]')
+toolsetVersions=$(jq -r '.toolcache[] | select(.name | contains("PyPy")) | .versions[]' $toolset)
 
 # Fail out if any setups fail
 set -e

images/linux/toolsets/toolset-1604.json

@@ -108,5 +108,86 @@
                 "4.6.0"
             ]
         }
+    ],
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl3",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libicu55",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
         ]
+    },
+    "docker": {
+        "images": [
+            "alpine:3.7",
+            "alpine:3.8",
+            "alpine:3.9",
+            "alpine:3.10",
+            "buildpack-deps:stretch",
+            "buildpack-deps:buster",
+            "debian:8",
+            "debian:9",
+            "jekyll/builder",
+            "mcr.microsoft.com/azure-pipelines/node8-typescript",
+            "node:10",
+            "node:12",
+            "node:10-alpine",
+            "node:12-alpine",
+            "ubuntu:14.04"
+        ]
+    }
 }

images/linux/toolsets/toolset-1804.json

@@ -104,5 +104,85 @@
                 "4.6.0"
             ]
         }
+    ],
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl3",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
         ]
+    },
+    "docker": {
+        "images": [
+            "alpine:3.7",
+            "alpine:3.8",
+            "alpine:3.9",
+            "alpine:3.10",
+            "buildpack-deps:stretch",
+            "buildpack-deps:buster",
+            "debian:8",
+            "debian:9",
+            "jekyll/builder",
+            "mcr.microsoft.com/azure-pipelines/node8-typescript",
+            "node:10",
+            "node:12",
+            "node:10-alpine",
+            "node:12-alpine",
+            "ubuntu:14.04"
+        ]
+    }
 }

images/linux/toolsets/toolset-2004.json

@@ -67,5 +67,86 @@
             "ndk-bundle",
             "platform-tools"
         ]
+    },
+    "apt": {
+        "common_packages": [
+            "dbus",
+            "dnsutils",
+            "dpkg",
+            "fakeroot",
+            "gnupg2",
+            "iproute2",
+            "iputils-ping",
+            "lib32z1",
+            "libc++abi-dev",
+            "libc++-dev",
+            "libcurl4",
+            "libgbm-dev",
+            "libgconf-2-4",
+            "libgtk-3-0",
+            "libsecret-1-dev",
+            "libsqlite3-dev",
+            "libunwind8",
+            "libxkbfile-dev",
+            "libxss1",
+            "locales",
+            "openssh-client",
+            "pkg-config",
+            "python-is-python2",
+            "rpm",
+            "texinfo",
+            "tk",
+            "tzdata",
+            "upx",
+            "xorriso",
+            "xvfb",
+            "xz-utils",
+            "zstd",
+            "zsync"
+        ],
+        "cmd_packages": [
+            "bison",
+            "brotli",
+            "bzip2",
+            "curl",
+            "file",
+            "flex",
+            "ftp",
+            "jq",
+            "m4",
+            "netcat",
+            "parallel",
+            "patchelf",
+            "rsync",
+            "shellcheck",
+            "sqlite3",
+            "ssh",
+            "sudo",
+            "telnet",
+            "time",
+            "unzip",
+            "wget",
+            "yamllint",
+            "zip"
+        ]
+    },
+    "docker": {
+        "images": [
+            "alpine:3.7",
+            "alpine:3.8",
+            "alpine:3.9",
+            "alpine:3.10",
+            "buildpack-deps:stretch",
+            "buildpack-deps:buster",
+            "debian:8",
+            "debian:9",
+            "jekyll/builder",
+            "mcr.microsoft.com/azure-pipelines/node8-typescript",
+            "node:10",
+            "node:12",
+            "node:10-alpine",
+            "node:12-alpine",
+            "ubuntu:14.04"
+        ]
     }
 }

images/linux/ubuntu1604.json

@@ -88,8 +88,19 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-1604.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
+            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}",
             "scripts":[
                 "{{template_dir}}/scripts/installers/preparemetadata.sh"
             ],
@@ -98,8 +109,7 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "ANNOUNCEMENTS={{user `announcements`}}"
-            ],
+            ]
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
         {
             "type": "shell",
@@ -198,31 +208,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1604.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-1604.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -232,7 +220,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu1804.json

@@ -91,8 +91,19 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-1804.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
+            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}",
             "scripts":[
                 "{{template_dir}}/scripts/installers/preparemetadata.sh"
             ],
@@ -101,8 +112,7 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "ANNOUNCEMENTS={{user `announcements`}}"
-            ],
+            ]
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
         {
             "type": "shell",
@@ -202,31 +212,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-1804.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-1804.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -236,7 +224,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/linux/ubuntu2004.json

@@ -93,8 +93,19 @@
             "source": "{{template_dir}}/scripts/installers",
             "destination": "{{user `installer_script_folder`}}"
         },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolcache-2004.json",
+            "destination": "{{user `installer_script_folder`}}/toolcache.json"
+        },
+        {
+            "type": "file",
+            "source": "{{template_dir}}/toolsets/toolset-2004.json",
+            "destination": "{{user `installer_script_folder`}}/toolset.json"
+        },
         {
             "type": "shell",
+            "execute_command": "chmod +x {{ .Path }}; sudo {{ .Vars }} {{ .Path }}",
             "scripts":[
                 "{{template_dir}}/scripts/installers/preparemetadata.sh"
             ],
@@ -103,8 +114,7 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "ANNOUNCEMENTS={{user `announcements`}}"
-            ],
+            ]
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },
         {
             "type": "shell",
@@ -204,31 +214,9 @@
                 "{{template_dir}}/scripts/installers/mongodb.sh",
                 "{{template_dir}}/scripts/installers/rndgenerator.sh",
                 "{{template_dir}}/scripts/installers/swig.sh",
-                "{{template_dir}}/scripts/installers/netlify.sh"
+                "{{template_dir}}/scripts/installers/netlify.sh",
-            ],
-            "environment_vars": [
-                "METADATA_FILE={{user `metadata_file`}}",
-                "HELPER_SCRIPTS={{user `helper_script_folder`}}",
-                "DEBIAN_FRONTEND=noninteractive"
-            ],
-            "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolcache-2004.json",
-            "destination": "{{user `installer_script_folder`}}/toolcache.json"
-        },
-        {
-            "type": "file",
-            "source": "{{template_dir}}/toolsets/toolset-2004.json",
-            "destination": "{{user `installer_script_folder`}}/toolset.json"
-        },
-        {
-            "type": "shell",
-            "scripts":[
                 "{{template_dir}}/scripts/installers/android.sh",
                 "{{template_dir}}/scripts/installers/azpowershell.sh",
-                "{{template_dir}}/scripts/helpers/containercache.sh",
                 "{{template_dir}}/scripts/installers/hosted-tool-cache.sh",
                 "{{template_dir}}/scripts/installers/pypy.sh",
                 "{{template_dir}}/scripts/installers/python.sh",
@@ -238,7 +226,8 @@
                 "METADATA_FILE={{user `metadata_file`}}",
                 "HELPER_SCRIPTS={{user `helper_script_folder`}}",
                 "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}",
-                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}"
+                "GITHUB_FEED_TOKEN={{user `github_feed_token`}}",
+                "DEBIAN_FRONTEND=noninteractive"
             ],
             "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'"
         },

images/macos/macos-10.15-Readme.md

@@ -1,7 +1,11 @@
-# macOS info
+| Announcements |
+|-|
+| [Xcode 11.7 will be set as a default one on September, 9](https://github.com/actions/virtual-environments/issues/1537) |
+***
+# macOS 10.15 info
 - System Version: macOS 10.15.6 (19G2021)
 - Kernel Version: Darwin 19.6.0
-- Image Version: 20200829.1
+- Image Version: 20200903.1
 
 ## Installed Software
 ### Language and Runtime
@@ -19,7 +23,7 @@
 - .NET SDK 2.1.300 2.1.301 2.1.302 2.1.401 2.1.402 2.1.403 2.1.500 2.1.502 2.1.503 2.1.504 2.1.505 2.1.506 2.1.507 2.1.602 2.1.603 2.1.604 2.1.607 2.1.700 2.1.701 2.1.801 2.1.802 2.1.803 2.1.804 2.1.805 2.1.806 2.1.807 2.1.808 2.1.809 3.0.100 3.0.101 3.0.102 3.0.103 3.1.100 3.1.101 3.1.200 3.1.201 3.1.300 3.1.301 3.1.302 3.1.401
 - R 4.0.2
 - Go 1.15
-- PHP 7.4.9
+- PHP 7.4.10
 - julia 1.5.1
 
 ### Package Management
@@ -27,7 +31,7 @@
 - Bundler version 2.1.4
 - Carthage 0.35.0
 - CocoaPods 1.9.3
-- Homebrew 2.4.14
+- Homebrew 2.4.16
 - NPM 6.14.6
 - Yarn 1.22.5
 - NuGet 5.6.0.6489
@@ -45,7 +49,7 @@
 ### Utilities
 - Curl 7.72.0
 - Git: 2.28.0
-- Git LFS: 2.11.0
+- Git LFS: 2.12.0
 - GitHub CLI: 0.11.1
 - Hub CLI: 2.14.2
 - GNU Wget 1.20.3
@@ -54,21 +58,21 @@
 - GNU parallel 20200722
 - OpenSSL 1.0.2t  10 Sep 2019 `(/usr/local/opt/openssl -> /usr/local/Cellar/openssl/1.0.2t)`
 - jq 1.6
-- gpg (GnuPG) 2.2.21
+- gpg (GnuPG) 2.2.22
 - psql (PostgreSQL) 12.4
 - PostgreSQL 12.4
 - aria2 1.35.0
 - azcopy 10.6.0
 - zstd 1.4.5
-- bazel 3.4.1
+- bazel 3.5.0
 - bazelisk 1.6.1
-- helm v3.3.0+g8a4aeec
+- helm v3.3.1+g249e521
 - virtualbox 6.1.12r139181
 - mongo v4.4.0
 - mongod v4.4.0
 - Vagrant 2.2.10
 - 7-Zip 16.02
-- Newman 5.1.2
+- Newman 5.2.0
 
 
 ### Tools
@@ -77,7 +81,7 @@
 - App Center CLI 2.6.7
 - Azure CLI 2.11.1
 - AWS CLI 2.0.44
-- AWS SAM CLI 1.1.0
+- AWS SAM CLI 1.2.0
 - AWS Session Manager CLI 1.1.61.0
 - Aliyun CLI 3.0.56
 - GHCup v0.1.10
@@ -94,9 +98,9 @@
 - SafariDriver 13.1.2 (15609.3.5.1.3)
 - Google Chrome 85.0.4183.83 
 - ChromeDriver 85.0.4183.87
-- Microsoft Edge 85.0.564.41 
+- Microsoft Edge 85.0.564.44 
-- MSEdgeDriver 85.0.564.41
+- MSEdgeDriver 85.0.564.44
-- Mozilla Firefox 80.0
+- Mozilla Firefox 80.0.1
 - geckodriver 0.27.0
 
 ### Java
@@ -136,8 +140,8 @@
 - 1.11.13
 - 1.12.17
 - 1.13.15
-- 1.14.7
+- 1.14.8
-- 1.15.0
+- 1.15.1
 
 ### Rust Tools
 - Rust 1.46.0
@@ -157,7 +161,7 @@
 | ---------- | ------- |
 | Az         | 4.6.1   |
 | MarkdownPS | 1.9     |
-| Pester     | 5.0.3   |
+| Pester     | 5.0.4   |
 
 ### Xamarin
 #### Visual Studio for Mac
@@ -204,10 +208,10 @@
 
 ### Xcode
 | Version        | Build    | Path                            |
-| -------------- | -------- | --------------------------------- |
+| -------------- | -------- | ------------------------------- |
 | 12.0 (beta)    | 12A8189n | /Applications/Xcode_12_beta.app |
-| 11.7 (beta)    | 11E801a  | /Applications/Xcode_11.7_beta.app |
+| 11.7 (default) | 11E801a  | /Applications/Xcode_11.7.app    |
-| 11.6 (default) | 11E708   | /Applications/Xcode_11.6.app      |
+| 11.6           | 11E708   | /Applications/Xcode_11.6.app    |
 | 11.5           | 11E608c  | /Applications/Xcode_11.5.app    |
 | 11.4.1         | 11E503a  | /Applications/Xcode_11.4.1.app  |
 | 11.4           | 11E146   | /Applications/Xcode_11.4.app    |

images/win/scripts/ImageHelpers/ImageHelpers.psm1

@@ -35,7 +35,8 @@ Export-ModuleMember -Function @(
     'Get-EnvironmentVariable'
     'Invoke-PesterTests'
     'Get-VsCatalogJsonPath'
+    'Get-VisualStudioPath'
     'Install-AndroidSDKPackages'
-    'Get-VisualStudioProduct'
+    'Get-VisualStudioPackages'
     'Get-VisualStudioComponents'
 )

images/win/scripts/ImageHelpers/InstallHelpers.ps1

@@ -258,6 +258,34 @@ function Install-VsixExtension
         }
 }
 
+function Get-VSExtensionVersion
+{
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [string] $packageName
+    )
+
+    $instanceFolders = Get-ChildItem -Path "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances"
+    if ($instanceFolders -is [array])
+    {
+        Write-Host "More than one instance installed"
+        exit 1
+    }
+
+    $stateContent = Get-Content -Path (Join-Path $instanceFolders.FullName '\state.packages.json')
+    $state = $stateContent | ConvertFrom-Json
+    $packageVersion = ($state.packages | Where-Object { $_.id -eq $packageName }).version
+
+    if (-not $packageVersion)
+    {
+        Write-Host "installed package $packageName for Visual Studio 2019 was not found"
+        exit 1
+    }
+
+    return $packageVersion
+}
+
 function Get-ToolcachePackages
 {
     $toolcachePath = Join-Path $env:ROOT_FOLDER "toolcache.json"

images/win/scripts/ImageHelpers/VisualStudioHelpers.ps1

@@ -60,60 +60,21 @@ Function Install-VisualStudio
     }
 }
 
-function Get-VisualStudioInstancePath {
-    return "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances\" + (Get-VisualStudioProduct -ProductType "VisualStudio").InstanceId
-}
-
 function Get-VsCatalogJsonPath {
-    return Join-Path (Get-VisualStudioInstancePath) "catalog.json"
+    $instanceFolder = Get-Item "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances\*" | Select-Object -First 1
+    return Join-Path $instanceFolder.FullName "catalog.json"
 }
 
-function Get-VisualStudioProduct {
+function Get-VisualStudioPath {
-    Param
+    return (Get-VSSetupInstance | Select-VSSetupInstance -Product *).InstallationPath
-    (
+}
-        [Parameter(Mandatory)]
-        [ValidateSet('VisualStudio','BuildTools')]
-        [String] $ProductType
-    )
 
-    if ($ProductType -eq "VisualStudio")
+function Get-VisualStudioPackages {
-    {
+    return (Get-VSSetupInstance | Select-VSSetupInstance -Product *).Packages
-        $VSSelectionType = "*Enterprise*"
-    }
-    elseif ($ProductType -eq "BuildTools")
-    {
-        $VSSelectionType = "*Build*"
-    }
-    return Get-VSSetupInstance | Where-Object -Property DisplayName -like $VSSelectionType
 }
 
 function Get-VisualStudioComponents {
-    Param
+    Get-VisualStudioPackages | Where-Object type -in 'Component', 'Workload' |
-    (
-        [Parameter(Mandatory)]
-        [String] $ProductType
-    )
-    (Get-VisualStudioProduct -ProductType $ProductType).Packages | Where-Object type -in 'Component', 'Workload' |
     Sort-Object Id, Version | Select-Object @{n = 'Package'; e = {$_.Id}}, Version |
     Where-Object { $_.Package -notmatch "[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}" }
 }
-
-function Get-VSExtensionVersion
-{
-    Param
-    (
-        [Parameter(Mandatory=$true)]
-        [string] $PackageName
-    )
-
-    $state = Get-Content -Path (Join-Path (Get-VisualStudioInstancePath) '\state.packages.json') | ConvertFrom-Json
-    $packageVersion = ($state.packages | Where-Object { $_.id -eq $PackageName }).version
-
-    if (-not $packageVersion)
-    {
-        Write-Host "installed package $PackageName for Visual Studio 2019 was not found"
-        exit 1
-    }
-
-    return $packageVersion
-}

images/win/scripts/Installers/Install-JavaTools.ps1

@@ -119,7 +119,7 @@ setx M2_REPO $m2_repo /M
 setx MAVEN_OPTS $maven_opts /M
 
 # Download cobertura jars
-$uri = 'https://ayera.dl.sourceforge.net/project/cobertura/cobertura/2.1.1/cobertura-2.1.1-bin.zip'
+$uri = 'https://downloads.sourceforge.net/project/cobertura/cobertura/2.1.1/cobertura-2.1.1-bin.zip'
 $coberturaPath = "C:\cobertura-2.1.1"
 
 $archivePath = Start-DownloadWithRetry -Url $uri -Name "cobertura.zip"

images/win/scripts/Installers/Install-NodeLts.ps1

@@ -17,11 +17,11 @@ Choco-Install -PackageName nodejs-lts -ArgumentList "--force"
 Add-MachinePathItem $PrefixPath
 $env:Path = Get-MachinePath
 
-setx NPM_CONFIG_PREFIX $PrefixPath /M
+setx npm_config_prefix $PrefixPath /M
-$env:NPM_CONFIG_PREFIX = $PrefixPath
+$env:npm_config_prefix = $PrefixPath
 
-setx NPM_CONFIG_CACHE $CachePath /M
+setx npm_config_cache $CachePath /M
-$env:NPM_CONFIG_CACHE = $CachePath
+$env:npm_config_cache = $CachePath
 
 npm config set registry http://registry.npmjs.org/
 

images/win/scripts/Installers/Install-VS.ps1

@@ -1,35 +1,38 @@
 ################################################################################
 ##  File:  Install-VS.ps1
-##  Desc:  Install Visual Studio and build tools
+##  Desc:  Install Visual Studio
 ################################################################################
 
 $ErrorActionPreference = "Stop"
 
 $toolset = Get-ToolsetContent
 $requiredComponents = $toolset.visualStudio.workloads | ForEach-Object { "--add $_" }
-$buildToolsRequiredComponents = $toolset.visualStudio.buildtools_workloads | ForEach-Object { "--add $_" }
 $workLoads = @(
 	"--allWorkloads --includeRecommended"
 	$requiredComponents
 	"--remove Component.CPython3.x64"
 )
 $workLoadsArgument = [String]::Join(" ", $workLoads)
-$buildToolsWorkloads= @(
-	"--includeRecommended"
-	$buildToolsRequiredComponents
-)
-$buildWorkLoadsArgument = [String]::Join(" ", $buildToolsWorkloads)
 
 $releaseInPath = $toolset.visualStudio.edition
 $subVersion = $toolset.visualStudio.subversion
 $bootstrapperUrl = "https://aka.ms/vs/${subVersion}/release/vs_${releaseInPath}.exe"
-$buildbootstrapperUrl = "https://aka.ms/vs/${subVersion}/release/vs_buildtools.exe"
 
-# Install VS and VS Build tools
+# Install VS
 Install-VisualStudio -BootstrapperUrl $bootstrapperUrl -WorkLoads $workLoadsArgument
-Install-VisualStudio -BootstrapperUrl $buildbootstrapperUrl -WorkLoads $buildWorkLoadsArgument
 
-$vsInstallRoot = (Get-VisualStudioProduct -ProductType "VisualStudio").InstallationPath
+# Find the version of VS installed for this instance
+# Only supports a single instance
+$vsProgramData = Get-Item -Path "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances"
+$instanceFolders = Get-ChildItem -Path $vsProgramData.FullName
+
+if ($instanceFolders -is [array])
+{
+    Write-Host "More than one instance installed"
+    exit 1
+}
+
+$vsInstallRoot = Get-VisualStudioPath
 
 # Initialize Visual Studio Experimental Instance
 & "$vsInstallRoot\Common7\IDE\devenv.exe" /RootSuffix Exp /ResetSettings General.vssettings /Command File.Exit

images/win/scripts/SoftwareReport/SoftwareReport.Generator.ps1

@@ -171,12 +171,7 @@ $markdown += New-MDNewLine
 
 $markdown += New-MDHeader "Workloads, components and extensions:" -Level 4
 $markdown += New-MDNewLine
-$markdown += ((Get-VisualStudioComponents -ProductType "VisualStudio") + (Get-VisualStudioExtensions)) | New-MDTable
+$markdown += ((Get-VisualStudioComponents) + (Get-VisualStudioExtensions)) | New-MDTable
-$markdown += New-MDNewLine
-
-$markdown += New-MDHeader "Build Tools Workloads:" -Level 4
-$markdown += New-MDNewLine
-$markdown += (Get-VisualStudioComponents -ProductType "BuildTools") | New-MDTable
 $markdown += New-MDNewLine
 
 $markdown += New-MDHeader "Microsoft Visual C++:" -Level 4

images/win/scripts/SoftwareReport/SoftwareReport.VisualStudio.psm1

@@ -23,18 +23,18 @@ function Get-VisualStudioExtensions {
     # Wix
     $vs = (Get-VisualStudioVersion).Name.Split()[-1]
     $wixPackageVersion = Get-WixVersion
-    $wixExtensionVersion = ((Get-VisualStudioProduct -ProductType "VisualStudio").Packages  | Where-Object {$_.Id -match 'WixToolset.VisualStudioExtension.Dev' -and $_.type -eq 'vsix'}).Version
+    $wixExtensionVersion = (Get-VisualStudioPackages | Where-Object {$_.Id -match 'WixToolset.VisualStudioExtension.Dev' -and $_.type -eq 'vsix'}).Version
 
     # WDK
-    $wdkPackageVersion = Get-VSExtensionVersion -PackageName 'Microsoft.Windows.DriverKit'
+    $wdkPackageVersion = Get-VSExtensionVersion -packageName 'Microsoft.Windows.DriverKit'
     $wdkExtensionVersion = Get-WDKVersion
 
     # SSDT
-    $analysisPackageVersion = Get-VSExtensionVersion -PackageName '04a86fc2-dbd5-4222-848e-911638e487fe'
+    $analysisPackageVersion = Get-VSExtensionVersion -packageName '04a86fc2-dbd5-4222-848e-911638e487fe'
-    $reportingPackageVersion = Get-VSExtensionVersion -PackageName '717ad572-c4b7-435c-c166-c2969777f718'
+    $reportingPackageVersion = Get-VSExtensionVersion -packageName '717ad572-c4b7-435c-c166-c2969777f718'
 
     $integrationPackageName = ($vs -match "2019") ? '851E7A09-7B2B-4F06-A15D-BABFCB26B970' : 'D1B09713-C12E-43CC-9EF4-6562298285AB'
-    $integrationPackageVersion = Get-VSExtensionVersion -PackageName $integrationPackageName
+    $integrationPackageVersion = Get-VSExtensionVersion -packageName $integrationPackageName
 
     $extensions = @(
         @{Package = 'SSDT Microsoft Analysis Services Projects'; Version = $analysisPackageVersion}

images/win/scripts/Tests/SSDTExtensions.Tests.ps1

@@ -9,12 +9,12 @@ Describe "SSDTExtensions" {
       )
 
       It "Extensions id=<id>" -TestCases $testExtenions {
-        $version = Get-VSExtensionVersion -PackageName "${id}"
+        $version = Get-VSExtensionVersion -packageName "${id}"
         $version | Should -Not -BeNullOrEmpty
       }
     } else {
       It "Extension SSDT" {
-        $version = Get-VSExtensionVersion -PackageName "SSDT"
+        $version = Get-VSExtensionVersion -packageName "SSDT"
         $version | Should -Not -BeNullOrEmpty
       }
     }

images/win/scripts/Tests/VisualStudio.Tests.ps1

@@ -5,27 +5,17 @@ Describe "Visual Studio" {
         }
 
         It "Devenv.exe" {
-            $vsInstallRoot = (Get-VisualStudioProduct -ProductType "VisualStudio").InstallationPath
+            $vsInstallRoot = Get-VisualStudioPath
             $devenvexePath = "${vsInstallRoot}\Common7\IDE\devenv.exe"
             $devenvexePath | Should -Exist
         }
     }
 
     Context "Visual Studio components" {
-        $testCases = (Get-ToolsetContent).visualStudio.workloads | ForEach-Object { @{ComponentName = $_} }
+        $expectedComponents = Get-ToolsetContent | Select-Object -ExpandProperty visualStudio | Select-Object -ExpandProperty workloads
+        $testCases = $expectedComponents | ForEach-Object { @{ComponentName = $_} }
         BeforeAll {
-            $installedComponents = Get-VisualStudioComponents -ProductType "VisualStudio" | Select-Object -ExpandProperty Package
+            $installedComponents = Get-VisualStudioComponents | Select-Object -ExpandProperty Package
-        }
-
-        It "<ComponentName>" -TestCases $testCases {
-            $installedComponents | Should -Contain $ComponentName
-        }
-    }
-
-    Context "Visual Studio Build Tools components" {
-        $testCases = (Get-ToolsetContent).visualStudio.buildtools_workloads | ForEach-Object { @{ComponentName = $_} }
-        BeforeAll {
-            $installedComponents = Get-VisualStudioComponents -ProductType "BuildTools" | Select-Object -ExpandProperty Package
         }
 
         It "<ComponentName>" -TestCases $testCases {

images/win/scripts/Tests/WDK.Tests.ps1

@@ -5,7 +5,7 @@ Describe "WDK" {
     }
 
     It "WDK version from system" {
-      $version = Get-VSExtensionVersion -PackageName "Microsoft.Windows.DriverKit"
+      $version = Get-VSExtensionVersion -packageName "Microsoft.Windows.DriverKit"
       $version | Should -Not -BeNullOrEmpty
     }
 }

images/win/scripts/Tests/Wix.Tests.ps1

@@ -12,11 +12,11 @@ Describe "Wix" {
     It "Wix Toolset version from system" {
       if (Test-IsWin19)
       {
-        $exVersion = Get-VSExtensionVersion -PackageName "WixToolset.VisualStudioExtension.Dev16"
+        $exVersion = Get-VSExtensionVersion -packageName "WixToolset.VisualStudioExtension.Dev16"
       }
       else
       {
-        $exVersion = Get-VSExtensionVersion -PackageName "WixToolset.VisualStudioExtension.Dev15"
+        $exVersion = Get-VSExtensionVersion -packageName "WixToolset.VisualStudioExtension.Dev15"
       }
       $exVersion | Should -Not -BeNullOrEmpty
     }

images/win/toolsets/toolset-2016.json

@@ -227,9 +227,6 @@
             "Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre",
             "Microsoft.VisualStudio.Component.Workflow",
             "Microsoft.VisualStudio.Workload.Office"
-        ],
-        "buildtools_workloads": [
-            "Microsoft.VisualStudio.Workload.WebBuildTools"
         ]
     }
 }

images/win/toolsets/toolset-2019.json

@@ -259,9 +259,6 @@
             "Microsoft.VisualStudio.Workload.VisualStudioExtension",
             "Component.MDD.Linux",
             "Component.MDD.Linux.GCC.arm"
-        ],
-        "buildtools_workloads": [
-            "Microsoft.VisualStudio.Workload.WebBuildTools"
         ]
     }
 }