From 9e55ec1fd40148c10aa6fad0b6f8e3c54182d613 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 23 Jul 2022 10:33:41 -0400 Subject: [PATCH] chore: improve devcontainer build (#4241) --- .devcontainer/devcontainer.json | 3 +- docker/app.Dockerfile | 23 +- docker/scripts/app-setup-debian.sh | 388 +++++++++++++++++++++++++++++ docker/scripts/app-setup-python.sh | 354 ++++++++++++++++++++++++++ 4 files changed, 752 insertions(+), 16 deletions(-) create mode 100644 docker/scripts/app-setup-debian.sh create mode 100644 docker/scripts/app-setup-python.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0fb24b880..68e8fc714 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,8 +14,7 @@ "features": { "docker-in-docker": { "version": "latest" - }, - "github-cli": "latest" + } }, // Set *default* container specific settings.json values on container create. diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index ca2ec1bb8..6d539b1a5 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -3,38 +3,33 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive -# Copy library scripts to execute -ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/ -ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/ -ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/ - -# [Option] Install zsh -ARG INSTALL_ZSH="true" -# [Option] Upgrade OS packages to their latest versions -ARG UPGRADE_PACKAGES="true" -# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies. +# Install needed packages and setup non-root user. ARG USERNAME=dev ARG USER_UID=1000 ARG USER_GID=$USER_UID +COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh +RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user - && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" + # Syntax: ./docker-setup-debian.sh [install zsh flag] [username] [user UID] [user GID] [upgrade packages flag] [install Oh My Zsh! flag] [Add non-free packages] + && bash /tmp/library-scripts/docker-setup-debian.sh "true" "${USERNAME}" "${USER_UID}" "${USER_GID}" "false" "true" "true" # Setup default python tools in a venv via pipx to avoid conflicts ENV PIPX_HOME=/usr/local/py-utils \ PIPX_BIN_DIR=/usr/local/py-utils/bin ENV PATH=${PATH}:${PIPX_BIN_DIR} -RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}" +COPY docker/scripts/app-setup-python.sh /tmp/library-scripts/docker-setup-python.sh +RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-python.sh && chmod +x /tmp/library-scripts/docker-setup-python.sh +RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}" # Remove library scripts for final image RUN rm -rf /tmp/library-scripts # Copy the startup file COPY docker/scripts/app-init.sh /docker-init.sh -RUN sed -i 's/\r$//' /docker-init.sh && \ - chmod +x /docker-init.sh +RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh # Fix user UID / GID to match host RUN groupmod --gid $USER_GID $USERNAME \ diff --git a/docker/scripts/app-setup-debian.sh b/docker/scripts/app-setup-debian.sh new file mode 100644 index 000000000..a66ba8197 --- /dev/null +++ b/docker/scripts/app-setup-debian.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +# +# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/common.md +# Maintainer: The VS Code and Codespaces Teams +# +# Syntax: ./common-debian.sh [install zsh flag] [username] [user UID] [user GID] [upgrade packages flag] [install Oh My Zsh! flag] [Add non-free packages] + +set -e + +INSTALL_ZSH=${1:-"true"} +USERNAME=${2:-"automatic"} +USER_UID=${3:-"automatic"} +USER_GID=${4:-"automatic"} +UPGRADE_PACKAGES=${5:-"true"} +INSTALL_OH_MYS=${6:-"true"} +ADD_NON_FREE_PACKAGES=${7:-"false"} +SCRIPT_DIR="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)" + +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi + +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# If in automatic mode, determine if a user already exists, if not use vscode +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=vscode + fi +elif [ "${USERNAME}" = "none" ]; then + USERNAME=root + USER_UID=0 + USER_GID=0 +fi + +# Load markers to see which steps have already run +if [ -f "${MARKER_FILE}" ]; then + echo "Marker file found:" + cat "${MARKER_FILE}" + source "${MARKER_FILE}" +fi + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +# Function to call apt-get if needed +apt_get_update_if_needed() +{ + if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update + else + echo "Skipping apt-get update." + fi +} + +# Run install apt-utils to avoid debconf warning then verify presence of other common developer tools and dependencies +if [ "${PACKAGES_ALREADY_INSTALLED}" != "true" ]; then + + package_list="apt-utils \ + openssh-client \ + gnupg2 \ + dirmngr \ + iproute2 \ + procps \ + lsof \ + htop \ + net-tools \ + psmisc \ + curl \ + wget \ + rsync \ + ca-certificates \ + unzip \ + zip \ + nano \ + vim-tiny \ + less \ + jq \ + lsb-release \ + apt-transport-https \ + dialog \ + libc6 \ + libgcc1 \ + libkrb5-3 \ + libgssapi-krb5-2 \ + libicu[0-9][0-9] \ + liblttng-ust[0-9] \ + libstdc++6 \ + zlib1g \ + locales \ + sudo \ + ncdu \ + man-db \ + strace \ + manpages \ + manpages-dev \ + init-system-helpers" + + # Needed for adding manpages-posix and manpages-posix-dev which are non-free packages in Debian + if [ "${ADD_NON_FREE_PACKAGES}" = "true" ]; then + # Bring in variables from /etc/os-release like VERSION_CODENAME + . /etc/os-release + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httredir)\.debian\.org\/debian ${VERSION_CODENAME} main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME} main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i -E "s/deb-src http:\/\/(deb|httpredir)\.debian\.org\/debian ${VERSION_CODENAME}-updates main/deb http:\/\/\1\.debian\.org\/debian ${VERSION_CODENAME}-updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}\/updates main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main/deb http:\/\/deb\.debian\.org\/debian ${VERSION_CODENAME}-backports main contrib non-free/" /etc/apt/sources.list + # Handle bullseye location for security https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html + sed -i "s/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + sed -i "s/deb-src http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main/deb http:\/\/security\.debian\.org\/debian-security ${VERSION_CODENAME}-security main contrib non-free/" /etc/apt/sources.list + echo "Running apt-get update..." + apt-get update + package_list="${package_list} manpages-posix manpages-posix-dev" + else + apt_get_update_if_needed + fi + + # Install libssl1.1 if available + if [[ ! -z $(apt-cache --names-only search ^libssl1.1$) ]]; then + package_list="${package_list} libssl1.1" + fi + + # Install appropriate version of libssl1.0.x if available + libssl_package=$(dpkg-query -f '${db:Status-Abbrev}\t${binary:Package}\n' -W 'libssl1\.0\.?' 2>&1 || echo '') + if [ "$(echo "$LIlibssl_packageBSSL" | grep -o 'libssl1\.0\.[0-9]:' | uniq | sort | wc -l)" -eq 0 ]; then + if [[ ! -z $(apt-cache --names-only search ^libssl1.0.2$) ]]; then + # Debian 9 + package_list="${package_list} libssl1.0.2" + elif [[ ! -z $(apt-cache --names-only search ^libssl1.0.0$) ]]; then + # Ubuntu 18.04, 16.04, earlier + package_list="${package_list} libssl1.0.0" + fi + fi + + echo "Packages to verify are installed: ${package_list}" + apt-get -y install --no-install-recommends ${package_list} 2> >( grep -v 'debconf: delaying package configuration, since apt-utils is not installed' >&2 ) + + # Install git if not already installed (may be more recent than distro version) + if ! type git > /dev/null 2>&1; then + apt-get -y install --no-install-recommends git + fi + + PACKAGES_ALREADY_INSTALLED="true" +fi + +# Get to latest versions of all packages +if [ "${UPGRADE_PACKAGES}" = "true" ]; then + apt_get_update_if_needed + apt-get -y upgrade --no-install-recommends + apt-get autoremove -y +fi + +# Ensure at least the en_US.UTF-8 UTF-8 locale is available. +# Common need for both applications and things like the agnoster ZSH theme. +if [ "${LOCALE_ALREADY_SET}" != "true" ] && ! grep -o -E '^\s*en_US.UTF-8\s+UTF-8' /etc/locale.gen > /dev/null; then + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + locale-gen + LOCALE_ALREADY_SET="true" +fi + +# Create or update a non-root user to match UID/GID. +group_name="${USERNAME}" +if id -u ${USERNAME} > /dev/null 2>&1; then + # User exists, update if needed + if [ "${USER_GID}" != "automatic" ] && [ "$USER_GID" != "$(id -g $USERNAME)" ]; then + group_name="$(id -gn $USERNAME)" + groupmod --gid $USER_GID ${group_name} + usermod --gid $USER_GID $USERNAME + fi + if [ "${USER_UID}" != "automatic" ] && [ "$USER_UID" != "$(id -u $USERNAME)" ]; then + usermod --uid $USER_UID $USERNAME + fi +else + # Create user + if [ "${USER_GID}" = "automatic" ]; then + groupadd $USERNAME + else + groupadd --gid $USER_GID $USERNAME + fi + if [ "${USER_UID}" = "automatic" ]; then + useradd -s /bin/bash --gid $USERNAME -m $USERNAME + else + useradd -s /bin/bash --uid $USER_UID --gid $USERNAME -m $USERNAME + fi +fi + +# Add add sudo support for non-root user +if [ "${USERNAME}" != "root" ] && [ "${EXISTING_NON_ROOT_USER}" != "${USERNAME}" ]; then + echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME + chmod 0440 /etc/sudoers.d/$USERNAME + EXISTING_NON_ROOT_USER="${USERNAME}" +fi + +# ** Shell customization section ** +if [ "${USERNAME}" = "root" ]; then + user_rc_path="/root" +else + user_rc_path="/home/${USERNAME}" +fi + +# Restore user .bashrc defaults from skeleton file if it doesn't exist or is empty +if [ ! -f "${user_rc_path}/.bashrc" ] || [ ! -s "${user_rc_path}/.bashrc" ] ; then + cp /etc/skel/.bashrc "${user_rc_path}/.bashrc" +fi + +# Restore user .profile defaults from skeleton file if it doesn't exist or is empty +if [ ! -f "${user_rc_path}/.profile" ] || [ ! -s "${user_rc_path}/.profile" ] ; then + cp /etc/skel/.profile "${user_rc_path}/.profile" +fi + +# .bashrc/.zshrc snippet +rc_snippet="$(cat << 'EOF' + +if [ -z "${USER}" ]; then export USER=$(whoami); fi +if [[ "${PATH}" != *"$HOME/.local/bin"* ]]; then export PATH="${PATH}:$HOME/.local/bin"; fi + +# Display optional first run image specific notice if configured and terminal is interactive +if [ -t 1 ] && [[ "${TERM_PROGRAM}" = "vscode" || "${TERM_PROGRAM}" = "codespaces" ]] && [ ! -f "$HOME/.config/vscode-dev-containers/first-run-notice-already-displayed" ]; then + if [ -f "/usr/local/etc/vscode-dev-containers/first-run-notice.txt" ]; then + cat "/usr/local/etc/vscode-dev-containers/first-run-notice.txt" + elif [ -f "/workspaces/.codespaces/shared/first-run-notice.txt" ]; then + cat "/workspaces/.codespaces/shared/first-run-notice.txt" + fi + mkdir -p "$HOME/.config/vscode-dev-containers" + # Mark first run notice as displayed after 10s to avoid problems with fast terminal refreshes hiding it + ((sleep 10s; touch "$HOME/.config/vscode-dev-containers/first-run-notice-already-displayed") &) +fi + +# Set the default git editor if not already set +if [ -z "$(git config --get core.editor)" ] && [ -z "${GIT_EDITOR}" ]; then + if [ "${TERM_PROGRAM}" = "vscode" ]; then + if [[ -n $(command -v code-insiders) && -z $(command -v code) ]]; then + export GIT_EDITOR="code-insiders --wait" + else + export GIT_EDITOR="code --wait" + fi + fi +fi + +EOF +)" + +# code shim, it fallbacks to code-insiders if code is not available +cat << 'EOF' > /usr/local/bin/code +#!/bin/sh + +get_in_path_except_current() { + which -a "$1" | grep -A1 "$0" | grep -v "$0" +} + +code="$(get_in_path_except_current code)" + +if [ -n "$code" ]; then + exec "$code" "$@" +elif [ "$(command -v code-insiders)" ]; then + exec code-insiders "$@" +else + echo "code or code-insiders is not installed" >&2 + exit 127 +fi +EOF +chmod +x /usr/local/bin/code + +# systemctl shim - tells people to use 'service' if systemd is not running +cat << 'EOF' > /usr/local/bin/systemctl +#!/bin/sh +set -e +if [ -d "/run/systemd/system" ]; then + exec /bin/systemctl/systemctl "$@" +else + echo '\n"systemd" is not running in this container due to its overhead.\nUse the "service" command to start services instead. e.g.: \n\nservice --status-all' +fi +EOF +chmod +x /usr/local/bin/systemctl + +# Codespaces bash and OMZ themes - partly inspired by https://github.com/ohmyzsh/ohmyzsh/blob/master/themes/robbyrussell.zsh-theme +codespaces_bash="$(cat \ +<<'EOF' + +# Codespaces bash prompt theme +__bash_prompt() { + local userpart='`export XIT=$? \ + && echo -n "\[\033[0;32m\]\u " \ + && [ "$XIT" -ne "0" ] && echo -n "\[\033[1;31m\]➜" || echo -n "\[\033[0m\]➜"`' + local lightblue='\[\033[1;34m\]' + local removecolor='\[\033[0m\]' + PS1="${userpart} ${lightblue}\w${removecolor}\$ " + unset -f __bash_prompt +} +__bash_prompt + +EOF +)" + +codespaces_zsh="$(cat \ +<<'EOF' +# Codespaces zsh prompt theme +__zsh_prompt() { + PROMPT="%{$fg[green]%}%n %(?:%{$reset_color%}➜ :%{$fg_bold[red]%}➜ )" # User/exit code arrow + PROMPT+='%{$fg_bold[blue]%}%(5~|%-1~/…/%3~|%4~)%{$reset_color%} ' # cwd + PROMPT+='%{$fg[white]%}$ %{$reset_color%}' + unset -f __zsh_prompt +} +ZSH_THEME_GIT_PROMPT_PREFIX="%{$fg_bold[cyan]%}(%{$fg_bold[red]%}" +ZSH_THEME_GIT_PROMPT_SUFFIX="%{$reset_color%} " +ZSH_THEME_GIT_PROMPT_DIRTY=" %{$fg_bold[yellow]%}✗%{$fg_bold[cyan]%})" +ZSH_THEME_GIT_PROMPT_CLEAN="%{$fg_bold[cyan]%})" +__zsh_prompt + +EOF +)" + +# Add RC snippet and custom bash prompt +if [ "${RC_SNIPPET_ALREADY_ADDED}" != "true" ]; then + echo "${rc_snippet}" >> /etc/bash.bashrc + echo "${codespaces_bash}" >> "${user_rc_path}/.bashrc" + echo 'export PROMPT_DIRTRIM=4' >> "${user_rc_path}/.bashrc" + if [ "${USERNAME}" != "root" ]; then + echo "${codespaces_bash}" >> "/root/.bashrc" + echo 'export PROMPT_DIRTRIM=4' >> "/root/.bashrc" + fi + chown ${USERNAME}:${group_name} "${user_rc_path}/.bashrc" + RC_SNIPPET_ALREADY_ADDED="true" +fi + +# Optionally install and configure zsh and Oh My Zsh! +if [ "${INSTALL_ZSH}" = "true" ]; then + if ! type zsh > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get install -y zsh + fi + if [ "${ZSH_ALREADY_INSTALLED}" != "true" ]; then + echo "${rc_snippet}" >> /etc/zsh/zshrc + ZSH_ALREADY_INSTALLED="true" + fi + + # Adapted, simplified inline Oh My Zsh! install steps that adds, defaults to a codespaces theme. + # See https://github.com/ohmyzsh/ohmyzsh/blob/master/tools/install.sh for official script. + oh_my_install_dir="${user_rc_path}/.oh-my-zsh" + if [ ! -d "${oh_my_install_dir}" ] && [ "${INSTALL_OH_MYS}" = "true" ]; then + template_path="${oh_my_install_dir}/templates/zshrc.zsh-template" + user_rc_file="${user_rc_path}/.zshrc" + umask g-w,o-w + mkdir -p ${oh_my_install_dir} + git clone --depth=1 \ + -c core.eol=lf \ + -c core.autocrlf=false \ + -c fsck.zeroPaddedFilemode=ignore \ + -c fetch.fsck.zeroPaddedFilemode=ignore \ + -c receive.fsck.zeroPaddedFilemode=ignore \ + "https://github.com/ohmyzsh/ohmyzsh" "${oh_my_install_dir}" 2>&1 + echo -e "$(cat "${template_path}")\nDISABLE_AUTO_UPDATE=true\nDISABLE_UPDATE_PROMPT=true" > ${user_rc_file} + sed -i -e 's/ZSH_THEME=.*/ZSH_THEME="codespaces"/g' ${user_rc_file} + + mkdir -p ${oh_my_install_dir}/custom/themes + echo "${codespaces_zsh}" > "${oh_my_install_dir}/custom/themes/codespaces.zsh-theme" + # Shrink git while still enabling updates + cd "${oh_my_install_dir}" + git repack -a -d -f --depth=1 --window=1 + # Copy to non-root user if one is specified + if [ "${USERNAME}" != "root" ]; then + cp -rf "${user_rc_file}" "${oh_my_install_dir}" /root + chown -R ${USERNAME}:${group_name} "${user_rc_path}" + fi + fi +fi + +echo "Done!" diff --git a/docker/scripts/app-setup-python.sh b/docker/scripts/app-setup-python.sh new file mode 100644 index 000000000..bdfc4d8bc --- /dev/null +++ b/docker/scripts/app-setup-python.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- +# +# Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/python.md +# Maintainer: The VS Code and Codespaces Teams +# +# Syntax: ./python-debian.sh [Python Version] [Python intall path] [PIPX_HOME] [non-root user] [Update rc files flag] [install tools flag] [Use Oryx if available flag] [Optimize when building from source flag] + +PYTHON_VERSION=${1:-"latest"} # 'system' checks the base image first, else installs 'latest' +PYTHON_INSTALL_PATH=${2:-"/usr/local/python"} +export PIPX_HOME=${3:-"/usr/local/py-utils"} +USERNAME=${4:-"automatic"} +UPDATE_RC=${5:-"true"} +INSTALL_PYTHON_TOOLS=${6:-"true"} +USE_ORYX_IF_AVAILABLE=${7:-"true"} +OPTIMIZE_BUILD_FROM_SOURCE=${8-"false"} + +DEFAULT_UTILS=("pylint" "flake8" "autopep8" "black" "yapf" "mypy" "pydocstyle" "pycodestyle" "bandit" "pipenv" "virtualenv") +PYTHON_SOURCE_GPG_KEYS="64E628F8D684696D B26995E310250568 2D347EA6AA65421D FB9921286F5E1540 3A5CA953F73C700D 04C367C218ADD4FF 0EDDC5F26A45C816 6AF053F07D9DC8D2 C9BE28DEE6DF025C 126EB563A74B06BF D9866941EA5BBD71 ED9D77D5" +GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com:80 +keyserver hkps://keys.openpgp.org +keyserver hkp://keyserver.pgp.com" + +set -e + +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi + +# Ensure that login shells get the correct path if the user updated the PATH using ENV. +rm -f /etc/profile.d/00-restore-env.sh +echo "export PATH=${PATH//$(sh -lc 'echo $PATH')/\$PATH}" > /etc/profile.d/00-restore-env.sh +chmod +x /etc/profile.d/00-restore-env.sh + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in ${POSSIBLE_USERS[@]}; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +updaterc() { + if [ "${UPDATE_RC}" = "true" ]; then + echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/bash.bashrc + fi + if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/zsh/zshrc + fi + fi +} + +# Get central common setting +get_common_setting() { + if [ "${common_settings_file_loaded}" != "true" ]; then + curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping." + common_settings_file_loaded=true + fi + if [ -f "/tmp/vsdc-settings.env" ]; then + local multi_line="" + if [ "$2" = "true" ]; then multi_line="-z"; fi + local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')" + if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi + fi + echo "$1=${!1}" +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + get_common_setting $1 + local keys=${!1} + get_common_setting GPG_KEY_SERVERS true + local keyring_args="" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + + # Use a temporary locaiton for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use Oryx to install something using a partial version match +oryx_install() { + local platform=$1 + local requested_version=$2 + local target_folder=${3:-none} + local ldconfig_folder=${4:-none} + echo "(*) Installing ${platform} ${requested_version} using Oryx..." + check_packages jq + # Soft match if full version not specified + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local version_list="$(oryx platforms --json | jq -r ".[] | select(.Name == \"${platform}\") | .Versions | sort | reverse | @tsv" | tr '\t' '\n' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$')" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + requested_version="$(echo "${version_list}" | head -n 1)" + else + set +e + requested_version="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + if [ -z "${requested_version}" ] || ! echo "${version_list}" | grep "^${requested_version//./\\.}$" > /dev/null 2>&1; then + echo -e "(!) Oryx does not support ${platform} version $2\nValid values:\n${version_list}" >&2 + return 1 + fi + echo "(*) Using ${requested_version} in place of $2." + fi + + export ORYX_ENV_TYPE=vsonline-present ORYX_PREFER_USER_INSTALLED_SDKS=true ENABLE_DYNAMIC_INSTALL=true DYNAMIC_INSTALL_ROOT_DIR=/opt + oryx prep --skip-detection --platforms-and-versions "${platform}=${requested_version}" + local opt_folder="/opt/${platform}/${requested_version}" + if [ "${target_folder}" != "none" ] && [ "${target_folder}" != "${opt_folder}" ]; then + ln -s "${opt_folder}" "${target_folder}" + fi + # Update library path add to conf + if [ "${ldconfig_folder}" != "none" ]; then + echo "/opt/${platform}/${requested_version}/lib" >> "/etc/ld.so.conf.d/${platform}.conf" + ldconfig + fi +} + +# Function to run apt-get if needed +apt_get_update_if_needed() +{ + if [ ! -d "/var/lib/apt/lists" ] || [ "$(ls /var/lib/apt/lists/ | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update + else + echo "Skipping apt-get update." + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install --no-install-recommends "$@" + fi +} + +install_from_source() { + if [ -d "${PYTHON_INSTALL_PATH}" ]; then + echo "(!) Path ${PYTHON_INSTALL_PATH} already exists. Remove this existing path or select a different one." + exit 1 + fi + echo "(*) Building Python ${PYTHON_VERSION} from source..." + # Install prereqs if missing + check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ + libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ + libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev + if ! type git > /dev/null 2>&1; then + apt_get_update_if_needed + apt-get -y install --no-install-recommends git + fi + + # Find version using soft match + find_version_from_git_tags PYTHON_VERSION "https://github.com/python/cpython" + + # Download tgz of source + mkdir -p /tmp/python-src "${PYTHON_INSTALL_PATH}" + cd /tmp/python-src + local tgz_filename="Python-${PYTHON_VERSION}.tgz" + local tgz_url="https://www.python.org/ftp/python/${PYTHON_VERSION}/${tgz_filename}" + echo "Downloading ${tgz_filename}..." + curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" + + # Verify signature + receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + echo "Downloading ${tgz_filename}.asc..." + curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" + gpg --verify "${tgz_filename}.asc" + + # Update min protocol for testing only - https://bugs.python.org/issue41561 + cp /etc/ssl/openssl.cnf /tmp/python-src/ + sed -i -E 's/MinProtocol[=\ ]+.*/MinProtocol = TLSv1.0/g' /tmp/python-src/openssl.cnf + export OPENSSL_CONF=/tmp/python-src/openssl.cnf + + # Untar and build + tar -xzf "/tmp/python-src/${tgz_filename}" -C "/tmp/python-src" --strip-components=1 + local config_args="" + if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then + config_args="--enable-optimizations" + fi + ./configure --prefix="${PYTHON_INSTALL_PATH}" --with-ensurepip=install ${config_args} + make -j 8 + make install + cd /tmp + rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env + chown -R ${USERNAME} "${PYTHON_INSTALL_PATH}" + ln -s ${PYTHON_INSTALL_PATH}/bin/python3 ${PYTHON_INSTALL_PATH}/bin/python + ln -s ${PYTHON_INSTALL_PATH}/bin/pip3 ${PYTHON_INSTALL_PATH}/bin/pip + ln -s ${PYTHON_INSTALL_PATH}/bin/idle3 ${PYTHON_INSTALL_PATH}/bin/idle + ln -s ${PYTHON_INSTALL_PATH}/bin/pydoc3 ${PYTHON_INSTALL_PATH}/bin/pydoc + ln -s ${PYTHON_INSTALL_PATH}/bin/python3-config ${PYTHON_INSTALL_PATH}/bin/python-config +} + +install_using_oryx() { + if [ -d "${PYTHON_INSTALL_PATH}" ]; then + echo "(!) Path ${PYTHON_INSTALL_PATH} already exists. Remove this existing path or select a different one." + exit 1 + fi + oryx_install "python" "${PYTHON_VERSION}" "${PYTHON_INSTALL_PATH}" "lib" || return 1 + ln -s ${PYTHON_INSTALL_PATH}/bin/idle3 ${PYTHON_INSTALL_PATH}/bin/idle + ln -s ${PYTHON_INSTALL_PATH}/bin/pydoc3 ${PYTHON_INSTALL_PATH}/bin/pydoc + ln -s ${PYTHON_INSTALL_PATH}/bin/python3-config ${PYTHON_INSTALL_PATH}/bin/python-config +} + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +# General requirements +check_packages curl ca-certificates gnupg2 tar make gcc libssl-dev zlib1g-dev libncurses5-dev \ + libbz2-dev libreadline-dev libxml2-dev xz-utils libgdbm-dev tk-dev dirmngr \ + libxmlsec1-dev libsqlite3-dev libffi-dev liblzma-dev uuid-dev + + +# Install python from source if needed +if [ "${PYTHON_VERSION}" != "none" ]; then + # If the os-provided versions are "good enough", detect that and bail out. + if [ ${PYTHON_VERSION} = "os-provided" ] || [ ${PYTHON_VERSION} = "system" ]; then + check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk + PYTHON_INSTALL_PATH="/usr" + should_install_from_source=false + elif [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then + install_using_oryx || should_install_from_source=true + else + should_install_from_source=true + fi + if [ "${should_install_from_source}" = "true" ]; then + install_from_source + fi + updaterc "if [[ \"\${PATH}\" != *\"${PYTHON_INSTALL_PATH}/bin\"* ]]; then export PATH=${PYTHON_INSTALL_PATH}/bin:\${PATH}; fi" +fi + +# If not installing python tools, exit +if [ "${INSTALL_PYTHON_TOOLS}" != "true" ]; then + echo "Done!" + exit 0 +fi + +export PIPX_BIN_DIR="${PIPX_HOME}/bin" +export PATH="${PYTHON_INSTALL_PATH}/bin:${PIPX_BIN_DIR}:${PATH}" + +# Create pipx group, dir, and set sticky bit +if ! cat /etc/group | grep -e "^pipx:" > /dev/null 2>&1; then + groupadd -r pipx +fi +usermod -a -G pipx ${USERNAME} +umask 0002 +mkdir -p ${PIPX_BIN_DIR} +chown :pipx ${PIPX_HOME} ${PIPX_BIN_DIR} +chmod g+s ${PIPX_HOME} ${PIPX_BIN_DIR} + +# Update pip if not using os provided python +if [ ${PYTHON_VERSION} != "os-provided" ] && [ ${PYTHON_VERSION} != "system" ]; then + echo "Updating pip..." + ${PYTHON_INSTALL_PATH}/bin/python3 -m pip install --no-cache-dir --upgrade pip +fi + +# Install tools +echo "Installing Python tools..." +export PYTHONUSERBASE=/tmp/pip-tmp +export PIP_CACHE_DIR=/tmp/pip-tmp/cache +pipx_path="" +if ! type pipx > /dev/null 2>&1; then + pip3 install --disable-pip-version-check --no-cache-dir --user pipx 2>&1 + /tmp/pip-tmp/bin/pipx install --pip-args=--no-cache-dir pipx + pipx_path="/tmp/pip-tmp/bin/" +fi +for util in ${DEFAULT_UTILS[@]}; do + if ! type ${util} > /dev/null 2>&1; then + ${pipx_path}pipx install --system-site-packages --pip-args '--no-cache-dir --force-reinstall' ${util} + else + echo "${util} already installed. Skipping." + fi +done +rm -rf /tmp/pip-tmp + +updaterc "$(cat << EOF +export PIPX_HOME="${PIPX_HOME}" +export PIPX_BIN_DIR="${PIPX_BIN_DIR}" +if [[ "\${PATH}" != *"\${PIPX_BIN_DIR}"* ]]; then export PATH="\${PATH}:\${PIPX_BIN_DIR}"; fi +EOF +)"