From ef87daf5af1de5d0031ad946f20e1601f1fb905e Mon Sep 17 00:00:00 2001 From: Josh Spicer Date: Tue, 16 Aug 2022 17:33:25 -0400 Subject: [PATCH] update `dotnet`/`oryx` features to use recommended install location for dotnet6 (#91) * use default jammy apt feed to mitigate jammy/oryx install issues * remove old comment * jammy first tries default apt repo * refactoring * glob friendly and tests * test files * better greps * correct path * DEBIAN_FRONTEND global in oryx install script * remove set -e from dotnet script to handle errors ourselves --- .devcontainer.json | 3 + .github/workflows/test-all.yaml | 2 +- .github/workflows/test-pr-temp.yaml | 15 --- .github/workflows/test-pr.yaml | 2 +- .github/workflows/test-scenarios.yaml | 2 +- src/dotnet/devcontainer-feature.json | 4 +- src/dotnet/install.sh | 66 ++++++++---- src/oryx/install.sh | 143 ++++++++++++++------------ test-scenarios/install_dotnet_3.sh | 13 +++ test-scenarios/install_dotnet_5.sh | 14 +++ test-scenarios/scenarios.json | 16 +++ test/dotnet/test.sh | 3 + 12 files changed, 178 insertions(+), 105 deletions(-) create mode 100644 .devcontainer.json delete mode 100644 .github/workflows/test-pr-temp.yaml create mode 100644 test-scenarios/install_dotnet_3.sh create mode 100644 test-scenarios/install_dotnet_5.sh diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 0000000..06cdde6 --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,3 @@ +{ + "postCreateCommand": "npm install -g @devcontainers/cli" +} \ No newline at end of file diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 49d38d7..f123cd8 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -8,7 +8,7 @@ on: jobs: test: runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')" + if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')" continue-on-error: true strategy: matrix: diff --git a/.github/workflows/test-pr-temp.yaml b/.github/workflows/test-pr-temp.yaml deleted file mode 100644 index 3f3dbe9..0000000 --- a/.github/workflows/test-pr-temp.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: "test" -on: - pull_request: - paths-ignore: - - ./**/oryx/** - - ./**/java/** - - ./**/dotnet/** - - ./**/php/** - - ./**/node/** - -jobs: - test: - runs-on: ubuntu-latest - steps: - - run: 'echo "No build required ; Temporary tweak for merging the PRs until branch protection rules are updated"' diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 04108a7..fce27c5 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -5,7 +5,7 @@ on: jobs: detect-changes: runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')" + if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')" outputs: features: ${{ steps.filter.outputs.changes }} steps: diff --git a/.github/workflows/test-scenarios.yaml b/.github/workflows/test-scenarios.yaml index 91fbf50..ba8a6c0 100644 --- a/.github/workflows/test-scenarios.yaml +++ b/.github/workflows/test-scenarios.yaml @@ -9,7 +9,7 @@ on: jobs: scenarios: runs-on: ubuntu-latest - if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore')" + if: "!contains(github.event.head_commit.message, 'no-ci') && !contains(github.event.head_commit.message, 'CI ignore') && !contains(github.event.head_commit.message, 'Automated documentation update')" steps: - uses: actions/checkout@v2 diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 156d1f0..5e11f11 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -8,8 +8,8 @@ "type": "string", "proposals": [ "latest", - "6.0", - "5.0", + "6", + "5", "3.1" ], "default": "latest", diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index 44fbe54..7cf38f1 100755 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -30,9 +30,6 @@ DOTNET_VERSION_CODENAMES_REQUIRE_OLDER_LIBSSL_1="buster bullseye bionic focal hi # alongside DOTNET_VERSION, but not set as default. ADDITIONAL_VERSIONS=${ADDITIONALVERSIONS:-""} -# Exit on failure. -set -e - # Setup STDERR. err() { echo "(!) $*" >&2 @@ -155,12 +152,12 @@ apt_cache_package_and_version_soft_match() { local exit_on_no_match="${3:-true}" local major_minor_version - major_minor_version="$(echo "${requested_version}" | cut -d "." --field=1,2)" - package_name="$(apt-cache search "${partial_package_name}-[0-9].[0-9]" | awk -F" - " '{print $1}' | grep -m 1 "${partial_package_name}-${major_minor_version}")" - # Ensure we've exported useful variables . /etc/os-release local architecture="$(dpkg --print-architecture)" + + major_minor_version="$(echo "${requested_version}" | cut -d "." --field=1,2)" + package_name="$(apt-cache search "${partial_package_name}-[0-9].[0-9]" | awk -F" - " '{print $1}' | grep -m 1 "${partial_package_name}-${major_minor_version}")" dot_escaped="${requested_version//./\\.}" dot_plus_escaped="${dot_escaped//+/\\+}" @@ -170,14 +167,14 @@ apt_cache_package_and_version_soft_match() { fuzzy_version="$(apt-cache madison ${package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${version_regex}")" set -e if [ -z "${fuzzy_version}" ]; then - echo "(!) No full or partial for package \"${package_name}\" match found in apt-cache for \"${requested_version}\" on OS ${ID} ${VERSION_CODENAME} (${architecture})." + echo "(!) No full or partial for package \"${partial_package_name}\" (resolved: \"${package_name}\") match found in apt-cache for \"${requested_version}\" on OS ${ID} ${VERSION_CODENAME} (${architecture})." if $exit_on_no_match; then echo "Available versions:" apt-cache madison ${package_name} | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+' exit 1 # Fail entire script else - echo "Continuing to fallback method (if available)" + echo "Continuing to fallback method if available" return 1; fi fi @@ -194,7 +191,7 @@ apt_cache_package_and_version_soft_match() { } # Install .NET CLI using apt-get package installer -install_using_apt() { +install_using_apt_from_microsoft_repo() { local sdk_or_runtime="$1" local dotnet_major_minor_version export DOTNET_PACKAGE="dotnet-${sdk_or_runtime}" @@ -206,7 +203,7 @@ install_using_apt() { get_common_setting MICROSOFT_GPG_KEYS_URI curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list - apt-get update + apt-get update -y if [ "${DOTNET_VERSION}" = "latest" ] || [ "${DOTNET_VERSION}" = "lts" ]; then DOTNET_VERSION="" @@ -214,18 +211,38 @@ install_using_apt() { else # Sets DOTNET_VERSION and DOTNET_PACKAGE if matches found. apt_cache_package_and_version_soft_match DOTNET_VERSION DOTNET_PACKAGE false + if [ "$?" != 0 ]; then + echo "Failed to find requested version." + return 1 + fi if [[ $(dotnet --version) == *"${DOTNET_VERSION}"* ]] ; then echo "Dotnet version ${DOTNET_VERSION} is already installed" - exit 1 - fi - - if [ "$?" != 0 ]; then return 1 fi + fi - if ! (apt-get install -yq ${DOTNET_PACKAGE}${DOTNET_VERSION}); then + echo "Installing '${DOTNET_PACKAGE}${DOTNET_VERSION}'..." + apt-get install -yq ${DOTNET_PACKAGE}${DOTNET_VERSION} + if [ "$?" != 0 ]; then + echo "Failed to complete apt install of ${DOTNET_PACKAGE}${DOTNET_VERSION}" + return 1 + fi +} + +install_using_default_apt_repo() { + DOTNET_PACKAGE="dotnet6" + + apt_get_update_if_needed + + if [[ "${DOTNET_VERSION}" = "latest" ]] || [[ "${DOTNET_VERSION}" = "lts" ]] || [[ ${DOTNET_VERSION} = "6"* ]]; then + if ! (apt-get install -yq ${DOTNET_PACKAGE}); then + echo "Failed to install 'dotnet6' package from default apt repo." + return 1 + fi + else + echo "The provided dotnet version is not distributed in this distro's default apt repo." return 1 fi } @@ -379,12 +396,15 @@ install_using_dotnet_releases_url() { export DEBIAN_FRONTEND=noninteractive -# Dotnet 3.1 and 5.0 are not supported on Ubuntu 22.04 (jammy)+, +. /etc/os-release +architecture="$(dpkg --print-architecture)" + +# Dotnet 3.1 and 5 are not supported on Ubuntu 22.04 (jammy)+, # due to lack of libssl3.0 support. # See: https://github.com/microsoft/vscode-dev-containers/issues/1458#issuecomment-1135077775 # NOTE: This will only guard against installation of the dotnet versions we propose via 'features'. # The user can attempt to install any other version at their own risk. -if [[ "${DOTNET_VERSION}" = "3.1" ]] || [[ "${DOTNET_VERSION}" = "5.0" ]]; then +if [[ "${DOTNET_VERSION}" = "3"* ]] || [[ "${DOTNET_VERSION}" = "5"* ]]; then if [[ ! "${DOTNET_VERSION_CODENAMES_REQUIRE_OLDER_LIBSSL_1}" = *"${VERSION_CODENAME}"* ]]; then err "Dotnet ${DOTNET_VERSION} is not supported on Ubuntu ${VERSION_CODENAME} due to a change in the 'libssl' dependency across distributions.\n Please upgrade your version of dotnet, or downgrade your OS version." exit 1 @@ -405,18 +425,20 @@ fi # Install the .NET CLI echo "(*) Installing .NET CLI..." -. /etc/os-release -architecture="$(dpkg --print-architecture)" - CHANGE_OWNERSHIP="false" if [[ "${DOTNET_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${DOTNET_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]] && [[ "${INSTALL_USING_APT}" = "true" ]]; then echo "Detected ${VERSION_CODENAME} on ${architecture}. Attempting to install dotnet from apt" - install_using_apt "${DOTNET_SDK_OR_RUNTIME}" + + install_using_default_apt_repo || install_using_apt_from_microsoft_repo "${DOTNET_SDK_OR_RUNTIME}" + if [ "$?" != 0 ]; then + echo "Could not install requested version from apt on current distribution." + exit 1 + fi else if [[ "${INSTALL_USING_APT}" = "false" ]]; then echo "Installing dotnet from releases url" else - echo "Could not install dotnet from apt. Attempting to install dotnet from releases url" + echo "Attempting to install dotnet from releases url" fi install_using_dotnet_releases_url "${DOTNET_SDK_OR_RUNTIME}" "${DOTNET_VERSION}" CHANGE_OWNERSHIP="true" diff --git a/src/oryx/install.sh b/src/oryx/install.sh index 0aeb8bd..a528ec7 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -3,14 +3,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. #------------------------------------------------------------------------------------------------------------- -# -# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/hugo.md -# Maintainer: The VS Code and Codespaces Teams + USERNAME=${USERNAME:-"automatic"} UPDATE_RC=${UPDATE_RC:-"true"} -set -eux +MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" + +set -eu if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' @@ -66,72 +66,89 @@ apt_get_update_if_needed() check_packages() { if ! dpkg -s "$@" > /dev/null 2>&1; then apt_get_update_if_needed - DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends "$@" + apt-get -y install --no-install-recommends "$@" fi } install_dotnet_using_apt() { - wget --no-check-certificate https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - rm packages-microsoft-prod.deb - - rm -rf /var/lib/apt/lists/* - check_packages apt-transport-https dotnet-sdk-6.0 + echo "Attempting to auto-install dotnet..." + install_from_microsoft_feed=false + apt_get_update_if_needed + apt-get -yq install dotnet6 || install_from_microsoft_feed="true" + + if [ "${install_from_microsoft_feed}" = "true" ]; then + echo "Attempting install from microsoft apt feed..." + curl -sSL ${MICROSOFT_GPG_KEYS_URI} | gpg --dearmor > /usr/share/keyrings/microsoft-archive-keyring.gpg + echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/microsoft-${ID}-${VERSION_CODENAME}-prod ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/microsoft.list + apt-get update -y + DOTNET_SKIP_FIRST_TIME_EXPERIENCE="true" apt-get install -yq dotnet-sdk-6.0 + fi + + echo -e "Finished attempt to install dotnet. Sdks installed:\n" + dotnet --list-sdks } -# Install dependencies -check_packages git sudo wget ca-certificates - # If we don't already have Oryx installed, install it now. -if ! oryx --version > /dev/null ; then - echo "Installing Oryx..." - - if ! cat /etc/group | grep -e "^oryx:" > /dev/null 2>&1; then - groupadd -r oryx - fi - usermod -a -G oryx "${USERNAME}" - - # Install dotnet unless available - if ! dotnet --version > /dev/null ; then - echo "'dotnet' was not detected. Attempting to install the latest version of the dotnet sdk to build oryx." - install_dotnet_using_apt - - if ! dotnet --version > /dev/null ; then - echo "Please install Dotnet before installing Oryx" - exit 1 - fi - fi - - BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen - ORYX=/usr/local/oryx - GIT_ORYX=/opt/tmp - - mkdir -p ${BUILD_SCRIPT_GENERATOR} - mkdir -p ${ORYX} - - git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX - - $GIT_ORYX/build/buildSln.sh - - dotnet publish -property:ValidateExecutableReferencesMatchSelfContained=false -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj - - dotnet publish -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildServer/BuildServer.csproj - - chmod a+x ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript - - ln -s ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript ${ORYX}/oryx - cp -f $GIT_ORYX/images/build/benv.sh ${ORYX}/benv - - ORYX_INSTALL_DIR="/opt" - mkdir -p "${ORYX_INSTALL_DIR}" - - updaterc "export ORYX_SDK_STORAGE_BASE_URL=https://oryx-cdn.microsoft.io && export ENABLE_DYNAMIC_INSTALL=true && DYNAMIC_INSTALL_ROOT_DIR=$ORYX_INSTALL_DIR && ORYX_PREFER_USER_INSTALLED_SDKS=true && export DEBIAN_FLAVOR=focal-scm" - - chown -R "${USERNAME}:oryx" "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}" - chmod -R g+r+w "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}" - find "${ORYX_INSTALL_DIR}" -type d | xargs -n 1 chmod g+s - find "${BUILD_SCRIPT_GENERATOR}" -type d | xargs -n 1 chmod g+s - find "${ORYX}" -type d | xargs -n 1 chmod g+s +if oryx --version > /dev/null ; then + echo "oryx is already installed. Skipping installation." + exit 0 fi +echo "Installing Oryx..." + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +. /etc/os-release +architecture="$(dpkg --print-architecture)" + +# Install dependencies +check_packages git sudo curl ca-certificates apt-transport-https gnupg2 dirmngr libc-bin + +if ! cat /etc/group | grep -e "^oryx:" > /dev/null 2>&1; then + groupadd -r oryx +fi +usermod -a -G oryx "${USERNAME}" + +# Install dotnet unless available +if ! dotnet --version > /dev/null ; then + echo "'dotnet' was not detected. Attempting to install the latest version of the dotnet sdk to build oryx." + install_dotnet_using_apt + + if ! dotnet --version > /dev/null ; then + echo "(!) Please install Dotnet before installing Oryx" + exit 1 + fi +fi + +BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen +ORYX=/usr/local/oryx +GIT_ORYX=/opt/tmp + +mkdir -p ${BUILD_SCRIPT_GENERATOR} +mkdir -p ${ORYX} + +git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX + +$GIT_ORYX/build/buildSln.sh + +dotnet publish -property:ValidateExecutableReferencesMatchSelfContained=false -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildScriptGeneratorCli/BuildScriptGeneratorCli.csproj +dotnet publish -r linux-x64 -o ${BUILD_SCRIPT_GENERATOR} -c Release $GIT_ORYX/src/BuildServer/BuildServer.csproj + +chmod a+x ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript + +ln -s ${BUILD_SCRIPT_GENERATOR}/GenerateBuildScript ${ORYX}/oryx +cp -f $GIT_ORYX/images/build/benv.sh ${ORYX}/benv + +ORYX_INSTALL_DIR="/opt" +mkdir -p "${ORYX_INSTALL_DIR}" + +updaterc "export ORYX_SDK_STORAGE_BASE_URL=https://oryx-cdn.microsoft.io && export ENABLE_DYNAMIC_INSTALL=true && DYNAMIC_INSTALL_ROOT_DIR=$ORYX_INSTALL_DIR && ORYX_PREFER_USER_INSTALLED_SDKS=true && export DEBIAN_FLAVOR=focal-scm" + +chown -R "${USERNAME}:oryx" "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}" +chmod -R g+r+w "${ORYX_INSTALL_DIR}" "${BUILD_SCRIPT_GENERATOR}" "${ORYX}" +find "${ORYX_INSTALL_DIR}" -type d | xargs -n 1 chmod g+s +find "${BUILD_SCRIPT_GENERATOR}" -type d | xargs -n 1 chmod g+s +find "${ORYX}" -type d | xargs -n 1 chmod g+s + echo "Done!" diff --git a/test-scenarios/install_dotnet_3.sh b/test-scenarios/install_dotnet_3.sh new file mode 100644 index 0000000..0d05215 --- /dev/null +++ b/test-scenarios/install_dotnet_3.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "dotnet sdks" dotnet --list-sdks +check "some major version of dotnet 3 is installed" dotnet --list-sdks | grep '3\.[0-9]*\.[0-9]*' +check "dotnet version 3 installed" ls -l /usr/share/dotnet/sdk | grep '3\.[0-9]*\.[0-9]*' + +# Report result +reportResults diff --git a/test-scenarios/install_dotnet_5.sh b/test-scenarios/install_dotnet_5.sh new file mode 100644 index 0000000..5f48eb1 --- /dev/null +++ b/test-scenarios/install_dotnet_5.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "dotnet sdks" dotnet --list-sdks +check "some major version of dotnet 5 is installed" dotnet --list-sdks | grep '5\.[0-9]*\.[0-9]*' +check "dotnet version 5 installed" ls -l /usr/share/dotnet/sdk | grep '5\.[0-9]*\.[0-9]*' + + +# Report result +reportResults diff --git a/test-scenarios/scenarios.json b/test-scenarios/scenarios.json index 3398df7..853da2f 100644 --- a/test-scenarios/scenarios.json +++ b/test-scenarios/scenarios.json @@ -75,5 +75,21 @@ "additionalVersions": "5.0,3.1.420" } } + }, + "install_dotnet_5": { + "image": "ubuntu:focal", + "features": { + "dotnet": { + "version": "5" + } + } + }, + "install_dotnet_3": { + "image": "ubuntu:focal", + "features": { + "dotnet": { + "version": "3" + } + } } } diff --git a/test/dotnet/test.sh b/test/dotnet/test.sh index ddba7a6..4e8281b 100755 --- a/test/dotnet/test.sh +++ b/test/dotnet/test.sh @@ -9,5 +9,8 @@ source dev-container-features-test-lib check "dotnet" dotnet --info check "sdks" dotnet --list-sdks +echo "Validating expected version present..." +check "some major version of dotnet 6 is installed" dotnet --list-sdks | grep '6\.[0-9]*\.[0-9]*' + # Report result reportResults \ No newline at end of file