update dotnet/oryx features to use recommended install location for dotnet6 (#91)

* use default jammy apt feed to mitigate jammy/oryx install issues * remove old comment * jammy first tries default apt repo * refactoring * glob friendly and tests * test files * better greps * correct path * DEBIAN_FRONTEND global in oryx install script * remove set -e from dotnet script to handle errors ourselves
2022-08-16 17:33:25 -04:00 · 2022-08-16 17:33:25 -04:00 · ef87daf5af
commit ef87daf5af
parent ea4eaf2a7a
12 changed files with 178 additions and 105 deletions
--- a/.devcontainer.json
+++ b/.devcontainer.json
@ -0,0 +1,3 @@
+{
+    "postCreateCommand": "npm install -g @devcontainers/cli"
+}
--- a/.github/workflows/test-all.yaml
+++ b/.github/workflows/test-all.yaml
@ -8,7 +8,7 @@ on:
 jobs:
  test:
    runs-on: ubuntu-latest
-    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')"
+    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')"
    continue-on-error: true
    strategy:
      matrix:
--- a/.github/workflows/test-pr-temp.yaml
+++ b/.github/workflows/test-pr-temp.yaml
@ -1,15 +0,0 @@
-name: "test"
-on:
-  pull_request:
-    paths-ignore:
-      - ./**/oryx/**
-      - ./**/java/**
-      - ./**/dotnet/**
-      - ./**/php/**
-      - ./**/node/**
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - run: 'echo "No build required ; Temporary tweak for merging the PRs until branch protection rules are updated"'
--- a/.github/workflows/test-pr.yaml
+++ b/.github/workflows/test-pr.yaml
@ -5,7 +5,7 @@ on:
 jobs:
  detect-changes:
    runs-on: ubuntu-latest
-    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')"
+    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')"
    outputs:
      features: ${{ steps.filter.outputs.changes }}
    steps:
--- a/.github/workflows/test-scenarios.yaml
+++ b/.github/workflows/test-scenarios.yaml
@ -9,7 +9,7 @@ on:
 jobs:
  scenarios:
    runs-on: ubuntu-latest
-    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')"
+    if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')"
    steps:
      - uses: actions/checkout@v2

--- a/src/dotnet/devcontainer-feature.json
+++ b/src/dotnet/devcontainer-feature.json
@ -8,8 +8,8 @@
            "type": "string",
            "proposals": [
                "latest",
-                "6.0",
-                "5.0",
+                "6",
+                "5",
                "3.1"
            ],
            "default": "latest",
--- a/src/dotnet/install.sh
+++ b/src/dotnet/install.sh
@ -30,9 +30,6 @@ DOTNET_VERSION_CODENAMES_REQUIRE_OLDER_LIBSSL_1="buster bullseye bionic focal hi
 # alongside DOTNET_VERSION, but not set as default.
 ADDITIONAL_VERSIONS=${ADDITIONALVERSIONS:-""}

-# Exit on failure.
-set -e
-
 # Setup STDERR.
 err() {
    echo "(!) $*" >&2
@ -155,13 +152,13 @@ apt_cache_package_and_version_soft_match() {
    local exit_on_no_match="${3:-true}"
    local major_minor_version

-    major_minor_version="$(echo "${requested_version}" | cut -d "." --field=1,2)"
-    package_name="$(apt-cache search "${partial_package_name}-[0-9].[0-9]" | awk -F" - " '{print $1}' | grep -m 1 "${partial_package_name}-${major_minor_version}")"
-
    # Ensure we've exported useful variables
    . /etc/os-release
    local architecture="$(dpkg --print-architecture)"

+    major_minor_version="$(echo "${requested_version}" | cut -d "." --field=1,2)"
+    package_name="$(apt-cache search "${partial_package_name}-[0-9].[0-9]" | awk -F" - " '{print $1}' | grep -m 1 "${partial_package_name}-${major_minor_version}")"
+    
    dot_escaped="${requested_version//./\\.}"
    dot_plus_escaped="${dot_escaped//+/\\+}"
    # Regex needs to handle debian package version number format: https://www.systutorials.com/docs/linux/man/5-deb-version/
@ -170,14 +167,14 @@ apt_cache_package_and_version_soft_match() {
        fuzzy_version="$(apt-cache madison ${package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${version_regex}")"
    set -e
    if [ -z "${fuzzy_version}" ]; then
-        echo "(!) No full or partial for package \"${package_name}\" match found in apt-cache for \"${requested_version}\" on OS ${ID} ${VERSION_CODENAME} (${architecture})."
+        echo "(!) No full or partial for package \"${partial_package_name}\" (resolved: \"${package_name}\") match found in apt-cache for \"${requested_version}\" on OS ${ID} ${VERSION_CODENAME} (${architecture})."

        if $exit_on_no_match; then
            echo "Available versions:"
            apt-cache madison ${package_name} | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+'
            exit 1 # Fail entire script
        else
-            echo "Continuing to fallback method (if available)"
+            echo "Continuing to fallback method if available"
            return 1;
        fi
    fi
@ -194,7 +191,7 @@ apt_cache_package_and_version_soft_match() {
 }

 # Install .NET CLI using apt-get package installer
-install_using_apt() {
+install_using_apt_from_microsoft_repo() {
    local sdk_or_runtime="$1"
    local dotnet_major_minor_version
    export DOTNET_PACKAGE="dotnet-${sdk_or_runtime}"
@ -206,7 +203,7 @@ install_using_apt() {
    get_common_setting MICROSOFT_GPG_KEYS_URI
    curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg
    echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list
-    apt-get update
+    apt-get update -y

    if [ "${DOTNET_VERSION}" = "latest" ] || [ "${DOTNET_VERSION}" = "lts" ]; then
        DOTNET_VERSION=""
@ -214,18 +211,38 @@ install_using_apt() {
    else
        # Sets DOTNET_VERSION and DOTNET_PACKAGE if matches found. 
        apt_cache_package_and_version_soft_match DOTNET_VERSION DOTNET_PACKAGE false
+        if [ "$?" != 0 ]; then
+            echo "Failed to find requested version."
+            return 1
+        fi

        if [[ $(dotnet --version) == *"${DOTNET_VERSION}"* ]] ; then
            echo "Dotnet version ${DOTNET_VERSION} is already installed"
-            exit 1
-        fi
-
-        if [ "$?" != 0 ]; then
            return 1
        fi
+
    fi

-    if ! (apt-get install -yq ${DOTNET_PACKAGE}${DOTNET_VERSION}); then
+    echo "Installing '${DOTNET_PACKAGE}${DOTNET_VERSION}'..."
+    apt-get install -yq ${DOTNET_PACKAGE}${DOTNET_VERSION}
+    if [ "$?" != 0 ]; then
+        echo "Failed to complete apt install of ${DOTNET_PACKAGE}${DOTNET_VERSION}"
+        return 1
+    fi
+}
+
+install_using_default_apt_repo() {
+    DOTNET_PACKAGE="dotnet6"
+
+    apt_get_update_if_needed
+
+    if [[ "${DOTNET_VERSION}" = "latest" ]] || [[ "${DOTNET_VERSION}" = "lts" ]] || [[ ${DOTNET_VERSION} = "6"* ]]; then
+        if ! (apt-get install -yq ${DOTNET_PACKAGE}); then
+            echo "Failed to install 'dotnet6' package from default apt repo."
+            return 1
+        fi
+    else
+        echo "The provided dotnet version is not distributed in this distro's default apt repo."
        return 1
    fi
 }
@ -379,12 +396,15 @@ install_using_dotnet_releases_url() {

 export DEBIAN_FRONTEND=noninteractive

-# Dotnet 3.1 and 5.0 are not supported on Ubuntu 22.04 (jammy)+,
+. /etc/os-release
+architecture="$(dpkg --print-architecture)"
+
+# Dotnet 3.1 and 5 are not supported on Ubuntu 22.04 (jammy)+,
 # due to lack of libssl3.0 support.
 # See: https://github.com/microsoft/vscode-dev-containers/issues/1458#issuecomment-1135077775
 # NOTE: This will only guard against installation of the dotnet versions we propose via 'features'. 
 #       The user can attempt to install any other version at their own risk.
-if [[ "${DOTNET_VERSION}" = "3.1" ]] || [[ "${DOTNET_VERSION}" = "5.0" ]]; then
+if [[ "${DOTNET_VERSION}" = "3"* ]] || [[ "${DOTNET_VERSION}" = "5"* ]]; then
    if [[ ! "${DOTNET_VERSION_CODENAMES_REQUIRE_OLDER_LIBSSL_1}" = *"${VERSION_CODENAME}"* ]]; then
        err "Dotnet ${DOTNET_VERSION} is not supported on Ubuntu ${VERSION_CODENAME} due to a change in the 'libssl' dependency across distributions.\n Please upgrade your version of dotnet, or downgrade your OS version."
        exit 1
@ -405,18 +425,20 @@ fi
 # Install the .NET CLI
 echo "(*) Installing .NET CLI..."

-. /etc/os-release
-architecture="$(dpkg --print-architecture)"
-
 CHANGE_OWNERSHIP="false"
 if [[ "${DOTNET_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[  "${DOTNET_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]] && [[ "${INSTALL_USING_APT}" = "true" ]]; then
    echo "Detected ${VERSION_CODENAME} on ${architecture}. Attempting to install dotnet from apt"
-    install_using_apt "${DOTNET_SDK_OR_RUNTIME}"
+
+    install_using_default_apt_repo || install_using_apt_from_microsoft_repo "${DOTNET_SDK_OR_RUNTIME}"
+    if [ "$?" != 0 ]; then
+        echo "Could not install requested version from apt on current distribution."
+        exit 1
+    fi
 else
    if [[ "${INSTALL_USING_APT}" = "false" ]]; then
        echo "Installing dotnet from releases url"
    else
-        echo "Could not install dotnet from apt. Attempting to install dotnet from releases url"
+        echo "Attempting to install dotnet from releases url"
    fi
    install_using_dotnet_releases_url "${DOTNET_SDK_OR_RUNTIME}" "${DOTNET_VERSION}"
    CHANGE_OWNERSHIP="true"
--- a/src/oryx/install.sh
+++ b/src/oryx/install.sh
@ -3,14 +3,14 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
 #-------------------------------------------------------------------------------------------------------------
-#
-# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/hugo.md
-# Maintainer: The VS Code and Codespaces Teams
+

 USERNAME=${USERNAME:-"automatic"}
 UPDATE_RC=${UPDATE_RC:-"true"}

-set -eux
+MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc"
+
+set -eu

 if [ "$(id -u)" -ne 0 ]; then
    echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
@ -66,72 +66,89 @@ apt_get_update_if_needed()
 check_packages() {
    if ! dpkg -s "$@" > /dev/null 2>&1; then
        apt_get_update_if_needed
-        DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends "$@"
+        apt-get -y install --no-install-recommends "$@"
    fi
 }

 install_dotnet_using_apt() {
-    wget --no-check-certificate https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-    sudo dpkg -i packages-microsoft-prod.deb
-    rm packages-microsoft-prod.deb
+    echo "Attempting to auto-install dotnet..."
+    install_from_microsoft_feed=false
+    apt_get_update_if_needed
+    apt-get -yq install dotnet6 || install_from_microsoft_feed="true"

-    rm -rf /var/lib/apt/lists/*
-    check_packages apt-transport-https dotnet-sdk-6.0
+    if [ "${install_from_microsoft_feed}" = "true" ]; then
+        echo "Attempting install from microsoft apt feed..."
+        curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg
+        echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list
+        apt-get update -y
+        DOTNET_SKIP_FIRST_TIME_EXPERIENCE="true" apt-get install -yq dotnet-sdk-6.0
+    fi
+
+    echo -e "Finished attempt to install dotnet.  Sdks installed:\n"
+    dotnet --list-sdks
 }

-# Install dependencies
-check_packages git sudo wget ca-certificates
-
 # If we don't already have Oryx installed, install it now.
-if ! oryx --version > /dev/null ; then
-    echo "Installing Oryx..."
+if  oryx --version > /dev/null ; then
+    echo "oryx is already installed. Skipping installation."
+    exit 0
+fi

-    if ! cat /etc/group | grep -e "^oryx:" > /dev/null 2>&1; then
+echo "Installing Oryx..."
+
+# Ensure apt is in non-interactive to avoid prompts
+export DEBIAN_FRONTEND=noninteractive
+
+. /etc/os-release
+architecture="$(dpkg --print-architecture)"
+
+# Install dependencies
+check_packages git sudo curl ca-certificates apt-transport-https gnupg2 dirmngr libc-bin
+
+if ! cat /etc/group | grep -e "^oryx:" > /dev/null 2>&1; then
    groupadd -r oryx
-    fi
-    usermod -a -G oryx "${USERNAME}"
+fi
+usermod -a -G oryx "${USERNAME}"

-    # Install dotnet unless available
-    if ! dotnet --version > /dev/null ; then
+# Install dotnet unless available
+if ! dotnet --version > /dev/null ; then
    echo "'dotnet' was not detected. Attempting to install the latest version of the dotnet sdk to build oryx."
    install_dotnet_using_apt

    if ! dotnet --version > /dev/null ; then
-            echo "Please install Dotnet before installing Oryx"
+        echo "(!) Please install Dotnet before installing Oryx"
        exit 1
    fi
-    fi
-
-    BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen 
-    ORYX=/usr/local/oryx
-    GIT_ORYX=/opt/tmp 
-
-    mkdir -p ${BUILD_SCRIPT_GENERATOR}
-    mkdir -p ${ORYX}
-
-    git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX
-
-    $GIT_ORYX/build/buildSln.sh
-
-    dotnet publish -property:ValidateExecutableReferencesMatchSelfContained=false -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj
-    
-    dotnet publish -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildServer/BuildServer.csproj
-
-    chmod a+x ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript
-
-    ln -s ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript ${ORYX}/oryx
-    cp -f $GIT_ORYX/images/build/benv.sh ${ORYX}/benv
-
-    ORYX_INSTALL_DIR="/opt"
-    mkdir -p "${ORYX_INSTALL_DIR}"
-
-    updaterc "export ORYX_SDK_STORAGE_BASE_URL=https://oryx-cdn.microsoft.io && export ENABLE_DYNAMIC_INSTALL=true && DYNAMIC_INSTALL_ROOT_DIR=$ORYX_INSTALL_DIR && ORYX_PREFER_USER_INSTALLED_SDKS=true && export DEBIAN_FLAVOR=focal-scm"
-    
-    chown -R "${USERNAME}:oryx" "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}"
-    chmod -R g+r+w "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}"
-    find "${ORYX_INSTALL_DIR}" -type d | xargs -n 1 chmod g+s
-    find "${BUILD_SCRIPT_GENERATOR}" -type d | xargs -n 1 chmod g+s
-    find "${ORYX}" -type d | xargs -n 1 chmod g+s
 fi

+BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen 
+ORYX=/usr/local/oryx
+GIT_ORYX=/opt/tmp
+
+mkdir -p ${BUILD_SCRIPT_GENERATOR}
+mkdir -p ${ORYX}
+
+git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX
+
+$GIT_ORYX/build/buildSln.sh
+
+dotnet publish -property:ValidateExecutableReferencesMatchSelfContained=false -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj
+dotnet publish -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildServer/BuildServer.csproj
+
+chmod a+x ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript
+
+ln -s ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript ${ORYX}/oryx
+cp -f $GIT_ORYX/images/build/benv.sh ${ORYX}/benv
+
+ORYX_INSTALL_DIR="/opt"
+mkdir -p "${ORYX_INSTALL_DIR}"
+
+updaterc "export ORYX_SDK_STORAGE_BASE_URL=https://oryx-cdn.microsoft.io && export ENABLE_DYNAMIC_INSTALL=true && DYNAMIC_INSTALL_ROOT_DIR=$ORYX_INSTALL_DIR && ORYX_PREFER_USER_INSTALLED_SDKS=true && export DEBIAN_FLAVOR=focal-scm"
+
+chown -R "${USERNAME}:oryx" "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}"
+chmod -R g+r+w "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}"
+find "${ORYX_INSTALL_DIR}" -type d | xargs -n 1 chmod g+s
+find "${BUILD_SCRIPT_GENERATOR}" -type d | xargs -n 1 chmod g+s
+find "${ORYX}" -type d | xargs -n 1 chmod g+s
+
 echo "Done!"
--- a/test-scenarios/install_dotnet_3.sh
+++ b/test-scenarios/install_dotnet_3.sh
@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+check "dotnet sdks" dotnet --list-sdks
+check "some major version of dotnet 3 is installed" dotnet --list-sdks |  grep '3\.[0-9]*\.[0-9]*'
+check "dotnet version 3 installed"  ls -l /usr/share/dotnet/sdk | grep '3\.[0-9]*\.[0-9]*'
+
+# Report result
+reportResults
--- a/test-scenarios/install_dotnet_5.sh
+++ b/test-scenarios/install_dotnet_5.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+check "dotnet sdks" dotnet --list-sdks
+check "some major version of dotnet 5 is installed" dotnet --list-sdks |  grep '5\.[0-9]*\.[0-9]*'
+check "dotnet version 5 installed"  ls -l /usr/share/dotnet/sdk | grep '5\.[0-9]*\.[0-9]*'
+
+
+# Report result
+reportResults
--- a/test-scenarios/scenarios.json
+++ b/test-scenarios/scenarios.json
@ -75,5 +75,21 @@
                "additionalVersions": "5.0,3.1.420"
            }
        }
+    },
+    "install_dotnet_5": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "5"
+            }
+        }
+    },
+    "install_dotnet_3": {
+        "image": "ubuntu:focal",
+        "features": {
+            "dotnet": {
+                "version": "3"
+            }
+        }
    }
 }
--- a/test/dotnet/test.sh
+++ b/test/dotnet/test.sh
@ -9,5 +9,8 @@ source dev-container-features-test-lib
 check "dotnet" dotnet --info
 check "sdks" dotnet --list-sdks

+echo "Validating expected version present..."
+check "some major version of dotnet 6 is installed" dotnet --list-sdks |  grep '6\.[0-9]*\.[0-9]*'
+
 # Report result
 reportResults