From b6186f1fab0bde0e58e3771f97f2301bd15b718a Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 20 May 2022 10:10:41 -0400 Subject: [PATCH] chore(dev): fix uid/gid mapping for non-vscode devs (#3981) * chore: fix uid/gid mapping for non-vscode devs * chore: fix volume ownership in devcontainer * chore: use zsh in init script + remove temp mounted folders * chore: ignore group mapping error during devcontainer build * chore: fix ownership when running as root on host * chore: use new docker compose command format + fix root check * chore: improve uid/gid mapping message output * chore: include chromedriver permission fix * chore: skip parent dev image and install node directly instead of nvm * chore: reapply gitignore changes + better dev help message --- .gitignore | 5 ++- .vscode/extensions.json | 5 +-- docker/app.Dockerfile | 67 ++++++++++++++++++++++++++++---------- docker/cleanall | 2 +- docker/cleandb | 8 ++--- docker/run | 52 +++++++++++++++++++++++++---- docker/scripts/app-init.sh | 18 +++++++--- 7 files changed, 117 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 433202a62..b028c9cb3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,9 +19,8 @@ datatracker.sublime-workspace /media /node_modules /release-coverage.json -/tmp -/tmp-nomcom-public-keys-dir -/tmp-storage-meetinghosttests-dir +/tmp-* +/.testresult *.pyc __pycache__ .yarn/* diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 8e8adf9ed..1612a87fe 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,3 @@ { - "recommendations": [ - "arcanis.vscode-zipfs", - "dbaeumer.vscode-eslint" - ] + "recommendations": [] } diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index e18c98aec..6d29ce5e0 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -1,31 +1,55 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3/.devcontainer/base.Dockerfile +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/python-3/.devcontainer/base.Dockerfile # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster -ARG VARIANT="3.10-bullseye" -FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} +ARG VARIANT=3-bullseye +FROM python:${VARIANT} LABEL maintainer="IETF Tools Team " -# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 -ARG NODE_VERSION="none" -RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi -RUN npm install -g yarn - -EXPOSE 8000 - ENV DEBIAN_FRONTEND=noninteractive +# Copy library scripts to execute +ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/ +ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/ +ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/ + +# [Option] Install zsh +ARG INSTALL_ZSH="true" +# [Option] Upgrade OS packages to their latest versions +ARG UPGRADE_PACKAGES="true" +# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies. +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID +RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 + && apt-get purge -y imagemagick imagemagick-6-common \ + # Install common packages, non-root user + && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" + +# Setup default python tools in a venv via pipx to avoid conflicts +ENV PIPX_HOME=/usr/local/py-utils \ + PIPX_BIN_DIR=/usr/local/py-utils/bin +ENV PATH=${PATH}:${PIPX_BIN_DIR} +RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}" + +# [Choice] Node.js version: lts, 18, 16, 14, 12, 10 +ARG NODE_VERSION="16" +RUN curl -fsSL "https://deb.nodesource.com/setup_${NODE_VERSION}.x" | bash - +RUN apt-get install -y nodejs make gcc g++ && npm install -g yarn + +# Remove library scripts for final image +RUN rm -rf /tmp/library-scripts + +# Expose port 8000 +EXPOSE 8000 + # Add Docker Source RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null -RUN apt-get update --fix-missing - -# apt-get upgrade is normally not a good idea, but this is a dev container -RUN apt-get -qy upgrade - # Install the packages we need -RUN apt-get install -qy \ +RUN apt-get update --fix-missing && apt-get install -qy \ apache2-utils \ apt-file \ apt-utils \ @@ -87,8 +111,11 @@ RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \ chmod +x /tmp/app-install-chromedriver.sh RUN /tmp/app-install-chromedriver.sh +# Fix /dev/shm permissions for chromedriver +RUN chmod 1777 /dev/shm + # Get rid of installation files we don't need in the image, to reduce size -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* # "fake" dbus address to prevent errors # https://github.com/SeleniumHQ/docker-selenium/issues/87 @@ -132,6 +159,12 @@ RUN sed -i 's/\r$//' /docker-init.sh && \ RUN mkdir -p /workspace WORKDIR /workspace +# Fix user UID / GID to match host +RUN groupmod --gid $USER_GID $USERNAME \ + && usermod --uid $USER_UID --gid $USER_GID $USERNAME \ + && chown -R $USER_UID:$USER_GID /home/$USERNAME \ + || exit 0 + USER vscode:vscode # Install current datatracker python dependencies diff --git a/docker/cleanall b/docker/cleanall index feb786a23..efe6f9f67 100755 --- a/docker/cleanall +++ b/docker/cleanall @@ -2,7 +2,7 @@ cd .. echo "Shutting down any instance still running and purge images..." -docker-compose down -v --rmi all +docker compose down -v --rmi all echo "Purging dangling images..." docker image prune cd docker diff --git a/docker/cleandb b/docker/cleandb index d94958261..4329a8601 100755 --- a/docker/cleandb +++ b/docker/cleandb @@ -2,9 +2,9 @@ echo "Shutting down any instance still running..." cd .. -docker-compose down -v +docker compose down -v echo "Rebuilding the DB image..." -docker-compose pull db -docker-compose build --no-cache db +docker compose pull db +docker compose build --no-cache db cd docker -echo "Done!" \ No newline at end of file +echo "Done!" diff --git a/docker/run b/docker/run index 9dc80ece5..5dd890751 100755 --- a/docker/run +++ b/docker/run @@ -3,18 +3,20 @@ # Usage info show_help() { cat << EOF -Usage: ${0##*/} [-h] [-p PORT] +Usage: ${0##*/} [-h] [-p PORT] [-r] Run datatracker in dev containers using docker-compose. -h display this help and exit -p PORT use custom HTTP port for datatracker + -r force rebuild the app container EOF } CUSTOM_PORT=8000 +FORCE_REBUILD=0 -while getopts hp: opt; do +while getopts "hp:r" opt; do case $opt in h) show_help @@ -24,6 +26,10 @@ while getopts hp: opt; do CUSTOM_PORT=$OPTARG echo "Using custom port $CUSTOM_PORT..." ;; + r) + FORCE_REBUILD=1 + echo "Will force rebuild the app container..." + ;; *) CUSTOM_PORT=8000 echo "Using port 8000..." @@ -31,12 +37,46 @@ while getopts hp: opt; do esac done +# Remove mounted temp directories +rm -rf .parcel-cache __pycache__ + +# Create extended docker-compose definition cp docker-compose.extend.yml docker-compose.extend-custom.yml sed -i -r -e "s/CUSTOM_PORT/$CUSTOM_PORT/" docker-compose.extend-custom.yml cd .. -docker-compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d -docker-compose port db 3306 -docker-compose exec app /bin/sh /docker-init.sh -docker-compose stop + +# Set UID/GID mappings +NEW_UID=$(id -u) +NEW_GID=$(id -g) +if [ $NEW_UID -gt 0 ]; then + echo "Will use the following user/group mapping:" + echo "USER ID: $NEW_UID" + echo "GROUP ID: $NEW_GID" +else + echo "Running as root, will use default user/group mapping..." + NEW_UID=1000 + NEW_GID=1000 +fi + +# Build / Rebuild Containers +if [ $FORCE_REBUILD == 1 ]; then + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml down + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml rm -f + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --no-cache --pull --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d --force-recreate +else + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID + docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d +fi + +# Output database port +echo "Database exposed on port:" +docker compose port db 3306 + +# Start init script +docker compose exec app /bin/zsh /docker-init.sh + +# Exit scripts +docker compose stop cd docker rm -f docker-compose.extend-custom.yml diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 0aad0b39b..becb376b4 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -4,9 +4,11 @@ WORKSPACEDIR="/workspace" sudo service rsyslog start -# fix permissions for npm-related paths -WORKSPACE_UID_GID=$(stat --format="%u:%g" "$WORKSPACEDIR") -sudo chown -R "$WORKSPACE_UID_GID" "$WORKSPACEDIR/.parcel-cache" +# Fix ownership of volumes +echo "Fixing volumes ownership..." +sudo chown -R vscode:vscode "$WORKSPACEDIR/.parcel-cache" +sudo chown -R vscode:vscode "$WORKSPACEDIR/__pycache__" +sudo chown -R vscode:vscode "$WORKSPACEDIR" # Build node packages that requrie native compilation echo "Compiling native node packages..." @@ -80,7 +82,9 @@ echo "Running initial checks..." /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local # /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local +echo "-----------------------------------------------------------------" echo "Done!" +echo "-----------------------------------------------------------------" if [ -z "$EDITOR_VSCODE" ]; then CODE=0 @@ -93,11 +97,15 @@ if [ -z "$EDITOR_VSCODE" ]; then echo echo "to start a development instance of the Datatracker." echo - bash + echo " ietf/manage.py test --settings=settings_sqlitetest" + echo + echo "to run all the tests." + echo + zsh else echo "Executing \"$*\" and stopping container." echo - bash -c "$*" + zsh -c "$*" CODE=$? fi sudo service rsyslog stop