chore(dev): fix uid/gid mapping for non-vscode devs (#3981)

* chore: fix uid/gid mapping for non-vscode devs

* chore: fix volume ownership in devcontainer

* chore: use zsh in init script + remove temp mounted folders

* chore: ignore group mapping error during devcontainer build

* chore: fix ownership when running as root on host

* chore: use new docker compose command format + fix root check

* chore: improve uid/gid mapping message output

* chore: include chromedriver permission fix

* chore: skip parent dev image and install node directly instead of nvm

* chore: reapply gitignore changes + better dev help message

.gitignore

@@ -19,9 +19,8 @@ datatracker.sublime-workspace
 /media
 /node_modules
 /release-coverage.json
-/tmp
+/tmp-*
-/tmp-nomcom-public-keys-dir
+/.testresult
-/tmp-storage-meetinghosttests-dir
 *.pyc
 __pycache__
 .yarn/*

.vscode/extensions.json

@@ -1,6 +1,3 @@
 {
-  "recommendations": [
+  "recommendations": []
-    "arcanis.vscode-zipfs",
-    "dbaeumer.vscode-eslint"
-  ]
 }

docker/app.Dockerfile

@@ -1,31 +1,55 @@
-# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3/.devcontainer/base.Dockerfile
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/python-3/.devcontainer/base.Dockerfile
 
 # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
-ARG VARIANT="3.10-bullseye"
+ARG VARIANT=3-bullseye
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+FROM python:${VARIANT}
 LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
 
-# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
-RUN npm install -g yarn
-
-EXPOSE 8000
-
 ENV DEBIAN_FRONTEND=noninteractive
 
+# Copy library scripts to execute
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/common-debian.sh /tmp/library-scripts/
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/python-debian.sh /tmp/library-scripts/
+ADD https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.236.0/containers/python-3/.devcontainer/library-scripts/meta.env /tmp/library-scripts/
+
+# [Option] Install zsh
+ARG INSTALL_ZSH="true"
+# [Option] Upgrade OS packages to their latest versions
+ARG UPGRADE_PACKAGES="true"
+# Install needed packages and setup non-root user. Use a separate RUN statement to add your own dependencies.
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
+    && apt-get purge -y imagemagick imagemagick-6-common \
+    # Install common packages, non-root user
+    && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true"
+
+# Setup default python tools in a venv via pipx to avoid conflicts
+ENV PIPX_HOME=/usr/local/py-utils \
+    PIPX_BIN_DIR=/usr/local/py-utils/bin
+ENV PATH=${PATH}:${PIPX_BIN_DIR}
+RUN bash /tmp/library-scripts/python-debian.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}"
+
+# [Choice] Node.js version: lts, 18, 16, 14, 12, 10
+ARG NODE_VERSION="16"
+RUN curl -fsSL "https://deb.nodesource.com/setup_${NODE_VERSION}.x" | bash -
+RUN apt-get install -y nodejs make gcc g++ && npm install -g yarn
+
+# Remove library scripts for final image
+RUN rm -rf /tmp/library-scripts
+
+# Expose port 8000
+EXPOSE 8000
+
 # Add Docker Source
 RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
 RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
     $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
 
-RUN apt-get update --fix-missing
-
-# apt-get upgrade is normally not a good idea, but this is a dev container
-RUN apt-get -qy upgrade
-
 # Install the packages we need
-RUN apt-get install -qy \
+RUN apt-get update --fix-missing && apt-get install -qy \
 	apache2-utils \
 	apt-file \
 	apt-utils \
@@ -87,8 +111,11 @@ RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \
     chmod +x /tmp/app-install-chromedriver.sh
 RUN /tmp/app-install-chromedriver.sh
 
+# Fix /dev/shm permissions for chromedriver
+RUN chmod 1777 /dev/shm
+
 # Get rid of installation files we don't need in the image, to reduce size
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 # "fake" dbus address to prevent errors
 # https://github.com/SeleniumHQ/docker-selenium/issues/87
@@ -132,6 +159,12 @@ RUN sed -i 's/\r$//' /docker-init.sh && \
 RUN mkdir -p /workspace
 WORKDIR /workspace
 
+# Fix user UID / GID to match host
+RUN groupmod --gid $USER_GID $USERNAME \
+    && usermod --uid $USER_UID --gid $USER_GID $USERNAME \
+    && chown -R $USER_UID:$USER_GID /home/$USERNAME \
+    || exit 0
+
 USER vscode:vscode
 
 # Install current datatracker python dependencies

docker/cleanall

@@ -2,7 +2,7 @@
 
 cd ..
 echo "Shutting down any instance still running and purge images..."
-docker-compose down -v --rmi all
+docker compose down -v --rmi all
 echo "Purging dangling images..."
 docker image prune
 cd docker

docker/cleandb

@@ -2,9 +2,9 @@
 
 echo "Shutting down any instance still running..."
 cd ..
-docker-compose down -v
+docker compose down -v
 echo "Rebuilding the DB image..."
-docker-compose pull db
+docker compose pull db
-docker-compose build --no-cache db
+docker compose build --no-cache db
 cd docker
-echo "Done!"
+echo "Done!"

docker/run

@@ -3,18 +3,20 @@
 # Usage info
 show_help() {
     cat << EOF
-Usage: ${0##*/} [-h] [-p PORT]
+Usage: ${0##*/} [-h] [-p PORT] [-r]
 Run datatracker in dev containers using docker-compose.
 
     -h          display this help and exit
     -p PORT     use custom HTTP port for datatracker
+    -r          force rebuild the app container
 
 EOF
 }
 
 CUSTOM_PORT=8000
+FORCE_REBUILD=0
 
-while getopts hp: opt; do
+while getopts "hp:r" opt; do
     case $opt in
         h)
             show_help
@@ -24,6 +26,10 @@ while getopts hp: opt; do
             CUSTOM_PORT=$OPTARG
             echo "Using custom port $CUSTOM_PORT..."
             ;;
+        r)
+            FORCE_REBUILD=1
+            echo "Will force rebuild the app container..."
+            ;;
         *)
             CUSTOM_PORT=8000
             echo "Using port 8000..."
@@ -31,12 +37,46 @@ while getopts hp: opt; do
     esac
 done
 
+# Remove mounted temp directories
+rm -rf .parcel-cache __pycache__
+
+# Create extended docker-compose definition
 cp docker-compose.extend.yml docker-compose.extend-custom.yml
 sed -i -r -e "s/CUSTOM_PORT/$CUSTOM_PORT/" docker-compose.extend-custom.yml
 cd ..
-docker-compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d
+
-docker-compose port db 3306
+# Set UID/GID mappings
-docker-compose exec app /bin/sh /docker-init.sh
+NEW_UID=$(id -u)
-docker-compose stop
+NEW_GID=$(id -g)
+if [ $NEW_UID -gt 0 ]; then
+    echo "Will use the following user/group mapping:"
+    echo "USER ID: $NEW_UID"
+    echo "GROUP ID: $NEW_GID"
+else
+    echo "Running as root, will use default user/group mapping..."
+    NEW_UID=1000
+    NEW_GID=1000
+fi
+
+# Build / Rebuild Containers
+if [ $FORCE_REBUILD ==  1 ]; then
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml down
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml rm -f
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --no-cache --pull --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d --force-recreate
+else
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml build --build-arg USER_UID=$NEW_UID --build-arg USER_GID=$NEW_GID
+    docker compose -f docker-compose.yml -f docker/docker-compose.extend-custom.yml up -d
+fi
+
+# Output database port
+echo "Database exposed on port:"
+docker compose port db 3306
+
+# Start init script
+docker compose exec app /bin/zsh /docker-init.sh
+
+# Exit scripts
+docker compose stop
 cd docker
 rm -f docker-compose.extend-custom.yml

docker/scripts/app-init.sh

@@ -4,9 +4,11 @@ WORKSPACEDIR="/workspace"
 
 sudo service rsyslog start
 
-# fix permissions for npm-related paths
+# Fix ownership of volumes
-WORKSPACE_UID_GID=$(stat --format="%u:%g" "$WORKSPACEDIR")
+echo "Fixing volumes ownership..."
-sudo chown -R "$WORKSPACE_UID_GID" "$WORKSPACEDIR/.parcel-cache"
+sudo chown -R vscode:vscode "$WORKSPACEDIR/.parcel-cache"
+sudo chown -R vscode:vscode "$WORKSPACEDIR/__pycache__"
+sudo chown -R vscode:vscode "$WORKSPACEDIR"
 
 # Build node packages that requrie native compilation
 echo "Compiling native node packages..."
@@ -80,7 +82,9 @@ echo "Running initial checks..."
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local
 # /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
 
+echo "-----------------------------------------------------------------"
 echo "Done!"
+echo "-----------------------------------------------------------------"
 
 if [ -z "$EDITOR_VSCODE" ]; then
     CODE=0
@@ -93,11 +97,15 @@ if [ -z "$EDITOR_VSCODE" ]; then
         echo
         echo "to start a development instance of the Datatracker."
         echo
-        bash
+        echo "    ietf/manage.py test --settings=settings_sqlitetest"
+        echo
+        echo "to run all the tests."
+        echo
+        zsh
     else
         echo "Executing \"$*\" and stopping container."
         echo
-        bash -c "$*"
+        zsh -c "$*"
         CODE=$?
     fi
     sudo service rsyslog stop