From 1e7b44787f1f06cefbcf91249b41904f32eb0c20 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 27 Oct 2021 16:41:02 +0000 Subject: [PATCH 1/6] misc: devcontainer configuration for vscode - Legacy-Id: 19477 --- .devcontainer/Dockerfile | 129 +++++++++++++++++++++++++ .devcontainer/devcontainer.json | 59 ++++++++++++ .devcontainer/docker-compose.yml | 56 +++++++++++ .devcontainer/init.sh | 156 +++++++++++++++++++++++++++++++ .devcontainer/settings_local.py | 65 +++++++++++++ .vscode/launch.json | 21 +++++ .vscode/settings.json | 11 +++ .vscode/tasks.json | 46 +++++++++ 8 files changed, 543 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml create mode 100755 .devcontainer/init.sh create mode 100644 .devcontainer/settings_local.py create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..799896369 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,129 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT="3.10-bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +EXPOSE 8000 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update +RUN apt-get -qy upgrade + +# Install the packages we need +RUN apt-get install -qy \ + build-essential \ + bzip2 \ + ca-certificates \ + colordiff \ + gawk \ + gcc \ + python3-ipython \ + jq \ + less \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + libgdbm-dev \ + libjpeg62-turbo-dev \ + liblzma-dev \ + libmagic1 \ + libmariadb-dev-compat \ + libmariadb-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libsvn1 \ + libxml2-dev \ + libxslt-dev \ + libyang1 \ + libz-dev \ + libffi-dev \ + locales \ + make \ + man \ + mariadb-client \ + netcat \ + openssh-client \ + patch \ + procps \ + pv \ + rsync \ + rsyslog \ + subversion \ + sudo \ + uuid-dev \ + vim \ + wget \ + xz-utils\ + zile \ + zlib1g-dev + +# Install libyang +# RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key && \ +# apt-key add - < Release.key && \ +# rm Release.key && \ +# echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list && \ +# apt-get update && \ +# apt-get install -qy libyang1 + +# Install chromedriver +RUN apt-get update && \ + apt-get install -y gnupg wget curl unzip --no-install-recommends && \ + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \ + apt-get update -y && \ + apt-get install -y google-chrome-stable && \ + CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \ + DRIVERVER=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") && \ + wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" && \ + unzip /chromedriver/chromedriver* -d /chromedriver && \ + ln -s /chromedriver/chromedriver /usr/local/bin/chromedriver && \ + ln -s /chromedriver/chromedriver /usr/bin/chromedriver + +# Get rid of installation files we don't need in the image, to reduce size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Enable some common locales +RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen && \ + locale-gen + +# Remove an rsyslog module that we don't need, which also requires extra permissions +RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf + +# Set up root password +RUN echo "root:root" | chpasswd + +# idnits and dependencies +ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/ +RUN chmod +rx /usr/local/bin/idnits + +# Directory for Mac certs +RUN mkdir /etc/certificates + +# Setup workspace +ENV HOSTNAME="datatracker" + +ENV DDIR="/usr/local/share/datatracker" +RUN mkdir -p $DDIR +WORKDIR $DDIR + +COPY requirements.txt /tmp/pip-tmp/ +RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ + && rm -rf /tmp/pip-tmp + +COPY .devcontainer/init.sh /docker-init.sh +RUN chmod +x /docker-init.sh +ENTRYPOINT ["/docker-init.sh"] +CMD ["sleep", "infinity"] + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..aee2e5d8a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,59 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3 +{ + "name": "IETF Datatracker", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/usr/local/share/datatracker", + "shutdownAction": "stopCompose", + + // Set *default* container specific settings.json values on container create. + "settings": { + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestArgs": [ + "ietf" + ], + "python.testing.unittestEnabled": true, + "python.testing.pytestEnabled": false, + "python.testing.unittestArgs": [ + "-v", + "-s", + "./ietf", + "-p", + "test*.py" + ] + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "editorconfig.editorconfig", + "redhat.vscode-yaml", + "visualstudioexptteam.vscodeintellicode", + "batisteo.vscode-django", + "littlefoxteam.vscode-python-test-adapter" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [8000, 3306], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "sh /docker-init.sh", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "django" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..1cdb8413f --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + # Append -bullseye or -buster to pin to an OS version. + # Use -bullseye variants on local arm64/Apple Silicon. + VARIANT: 3.6-bullseye + + volumes: + - ..:/usr/local/share/datatracker:cached + init: true + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db + + depends_on: + - db + + environment: + USER: django + UID: 1001 + GID: 1001 + DATADIR: data + + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: mariadb:10 + restart: unless-stopped + volumes: + - mariadb-data:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: ietf + MYSQL_DATABASE: ietf_utf8 + MYSQL_USER: django + MYSQL_PASSWORD: RkTkDPFnKpko + command: ['--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci'] + + + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + +volumes: + mariadb-data: \ No newline at end of file diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh new file mode 100755 index 000000000..0faed96ab --- /dev/null +++ b/.devcontainer/init.sh @@ -0,0 +1,156 @@ +#!/bin/bash + +# A little bit of setup +export LANG=en_GB.UTF-8 + +WORKSPACEDIR="/usr/local/share/datatracker" + +echo "Gathering info ..." +if [ ! "$USER" ]; then + echo "Environment variable USER is not set -- will set USER='django'." + USER="django" +fi +if [ ! "$UID" ]; then + echo "Environment variable UID is not set -- will set UID='1000'." + UID="1000" +fi +if [ ! "$GID" ]; then + echo "Environment variable GID is not set -- will set GID='1000'." + GID="1000" +fi +if [ ! "$TAG" ]; then + echo "Environment variable TAG is not set -- will set TAG='datatracker'." + TAG="datatracker" +fi +echo "User $USER ($UID:$GID)" + +echo "Checking if syslogd is running ..." +if ! /etc/init.d/rsyslog status > /dev/null; then + echo "Starting syslogd ..." + /etc/init.d/rsyslog start +fi + +echo "Waiting for DB container to come online ..." +wget -qO- https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for | sh -s -- localhost:3306 -- echo "DB ready" + +echo "Checking if the IETF database exists in DB container ..." +if ! mysql --protocol tcp -h localhost -u root --password=ietf --database="ietf_utf8" --execute="SHOW TABLES;" | grep -q 'django'; then + echo "Fetching database ..." + DUMPDIR=/home/$USER/$DATADIR + wget -N --progress=bar:force:noscroll -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz + + echo "Creating database ..." + mysql --protocol tcp -h localhost -u root --password=ietf --database="ietf_utf8" --execute="DROP DATABASE IF EXISTS ietf_utf8;" + mysqladmin --protocol tcp -h localhost -u root --password=ietf --default-character-set=utf8 create ietf_utf8 + + echo "Setting up permissions ..." + mysql --protocol tcp -h localhost -u root --password="ietf" ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO 'django'@'%' IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;" + + echo "Loading database (this may take a while)..." + gunzip < $DUMPDIR/ietf_utf8.sql.gz \ + | pv --progress --bytes --rate --eta --cursor --force --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \ + | sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \ + | mysql --protocol tcp -h localhost -u django --password=RkTkDPFnKpko -s -f ietf_utf8 \ + && rm $DUMPDIR/ietf_utf8.sql.gz +fi + +if ! grep -q ":$GID:$" /etc/group ; then + echo "Creating group entry for GID '$GID' ..." + groupadd -g "$GID" "$USER" +fi +if ! id -u "$USER" &> /dev/null; then + echo "Creating user '$USER' ..." + useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER + echo "$USER:$USER" | chpasswd +fi + +VIRTDIR="/opt/home/$USER/$TAG" +echo "Checking that there's a virtual environment for $TAG ..." +if [ ! -f $VIRTDIR/bin/activate ]; then + echo "Setting up python virtualenv at $VIRTDIR ..." + mkdir -p $VIRTDIR + python3.6 -m venv $VIRTDIR + echo -e " +# This is from $VIRTDIR/bin/activate, to activate the +# datatracker virtual python environment on docker container entry: +" >> /etc/bash.bashrc + cat $VIRTDIR/bin/activate >> /etc/bash.bashrc + cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc +else + echo "Using virtual environment at $VIRTDIR" +fi + +echo "Activating the virtual python environment ..." +. $VIRTDIR/bin/activate + +if [ ! -f $WORKSPACEDIR/ietf/settings_local.py ]; then + echo "Setting up a default settings_local.py ..." + cp $WORKSPACEDIR/.devcontainer/settings_local.py $WORKSPACEDIR/ietf/settings_local.py +fi + +for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do + dir="$WORKSPACEDIR/$sub" + if [ ! -d "$dir" ]; then + echo "Creating dir $dir" + mkdir -p "$dir"; + fi +done + +for sub in \ + nomcom_keys/public_keys \ + developers/ietf-ftp \ + developers/ietf-ftp/bofreq \ + developers/ietf-ftp/charter \ + developers/ietf-ftp/conflict-reviews \ + developers/ietf-ftp/internet-drafts \ + developers/ietf-ftp/rfc \ + developers/ietf-ftp/status-changes \ + developers/ietf-ftp/yang/catalogmod \ + developers/ietf-ftp/yang/draftmod \ + developers/ietf-ftp/yang/ianamod \ + developers/ietf-ftp/yang/invalmod \ + developers/ietf-ftp/yang/rfcmod \ + developers/www6s \ + developers/www6s/staging \ + developers/www6s/wg-descriptions \ + developers/www6s/proceedings \ + developers/www6/ \ + developers/www6/iesg \ + developers/www6/iesg/evaluation \ + ; do + dir="$WORKSPACEDIR/data/$sub" + if [ ! -d "$dir" ]; then + echo "Creating dir $dir" + mkdir -p "$dir"; + chown "$USER" "$dir" + fi +done + +if [ ! -f "$WORKSPACEDIR/test/data/draft-aliases" ]; then + echo "Generating draft aliases ..." + ietf/bin/generate-draft-aliases } +fi + +if [ ! -f "$WORKSPACEDIR/test/data/group-aliases" ]; then + echo "Generating group aliases ..." + ietf/bin/generate-wg-aliases } +fi + +chown -R $USER /opt/home/$USER +chmod -R g+w /usr/local/lib/ # so we can patch libs if needed + +cd "$WORKSPACEDIR" || cd "/home/$USER/" + +echo "Done!" +if ! echo "$LANG" | grep "UTF-8"; then + echo "" + echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc" +else + echo "LANG=$LANG" +fi + +HOME=/opt/home/$USER + +# su -p $USER + +exec "$@" \ No newline at end of file diff --git a/.devcontainer/settings_local.py b/.devcontainer/settings_local.py new file mode 100644 index 000000000..708ac61f9 --- /dev/null +++ b/.devcontainer/settings_local.py @@ -0,0 +1,65 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +import six +if six.PY3: + from typing import Collection, Dict, List, Tuple # pyflakes:ignore + +ALLOWED_HOSTS = ['*'] + +SECRET_KEY = 'jzv$o93h_lzw4a0%0oz-5t5lk+ai=3f8x@uo*9ahu8w4i300o6' + +DATABASES = { + 'default': { + 'HOST': 'localhost', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +} # type: Dict[str, Dict[str, Collection[str]]] + +DATABASE_TEST_OPTIONS = { + 'init_command': 'SET storage_engine=InnoDB', + } + +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" +IDSUBMIT_REPOSITORY_PATH = "test/id/" +IDSUBMIT_STAGING_PATH = "test/staging/" +INTERNET_DRAFT_ARCHIVE_DIR = "test/archive/" +INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "test/archive/" +RFC_PATH = "test/rfc/" + +AGENDA_PATH = 'data/developers/www6s/proceedings/' +MEETINGHOST_LOGO_PATH = AGENDA_PATH + +USING_DEBUG_EMAIL_SERVER=True +EMAIL_HOST='localhost' +EMAIL_PORT=2025 + +TRAC_WIKI_DIR_PATTERN = "test/wiki/%s" +TRAC_SVN_DIR_PATTERN = "test/svn/%s" +TRAC_CREATE_ADHOC_WIKIS = [ +] # type: List[Tuple[str, str, str]] + +MEDIA_BASE_DIR = 'test' +MEDIA_ROOT = MEDIA_BASE_DIR + '/media/' +MEDIA_URL = '/media/' + +PHOTOS_DIRNAME = 'photo' +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +DOCUMENT_PATH_PATTERN = 'data/developers/ietf-ftp/{doc.type_id}/' + +SUBMIT_YANG_CATALOG_MODEL_DIR = 'data/developers/ietf-ftp/yang/catalogmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = 'data/developers/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = 'data/developers/ietf-ftp/yang/invalmod/' +SUBMIT_YANG_IANA_MODEL_DIR = 'data/developers/ietf-ftp/yang/ianamod/' +SUBMIT_YANG_RFC_MODEL_DIR = 'data/developers/ietf-ftp/yang/rfcmod/' + + diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..fb7ccb05a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Run Server", + "type": "python", + "request": "launch", + "python": "/usr/local/bin/python", + "program": "${workspaceFolder}/ietf/manage.py", + "args": [ + "runserver", + "0.0.0.0:8000", + "--settings=settings_local" + ], + "django": true + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..3e5e7f8dd --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.testing.unittestArgs": [ + "-v", + "-s", + "./ietf", + "-p", + "test*.py" + ], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..07e68c657 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,46 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Run Checks", + "type": "shell", + "command": "/usr/local/bin/python", + "args": [ + "${workspaceFolder}/ietf/manage.py", + "check", + "--settings=settings_local" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + }, + { + "label": "Run Tests", + "type": "shell", + "command": "/usr/local/bin/python", + "args": [ + "${workspaceFolder}/ietf/manage.py", + "test", + "--settings=settings_sqlitetest" + ], + "group": "test", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "new", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + } + ] +} \ No newline at end of file From fe502da612424cffb583f39e84660c4c05710c63 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 27 Oct 2021 18:44:19 +0000 Subject: [PATCH 2/6] misc: devcontainer docker setup fix - Legacy-Id: 19479 --- .devcontainer/Dockerfile | 6 ++++-- .devcontainer/devcontainer.json | 1 + .vscode/tasks.json | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 799896369..640444d41 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get -qy upgrade # Install the packages we need RUN apt-get install -qy \ + bash \ build-essential \ bzip2 \ ca-certificates \ @@ -117,8 +118,9 @@ RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requ && rm -rf /tmp/pip-tmp COPY .devcontainer/init.sh /docker-init.sh -RUN chmod +x /docker-init.sh -ENTRYPOINT ["/docker-init.sh"] +RUN sed -i 's/\r$//' /docker-init.sh && \ + chmod +x /docker-init.sh +# ENTRYPOINT ["/usr/local/share/datatracker/.devcontainer/init.sh"] CMD ["sleep", "infinity"] # [Optional] Uncomment this section to install additional OS packages. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index aee2e5d8a..7574cbc0e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,6 +6,7 @@ "service": "app", "workspaceFolder": "/usr/local/share/datatracker", "shutdownAction": "stopCompose", + "postCreateCommand": "/docker-init.sh", // Set *default* container specific settings.json values on container create. "settings": { diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 07e68c657..1319ef547 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -41,6 +41,23 @@ "clear": false }, "problemMatcher": [] + }, + { + "label": "Re-run Setup Project", + "type": "shell", + "command": "/bin/bash", + "args": [ + "/docker-init.sh" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "new", + "showReuseMessage": false, + "clear": false + }, + "problemMatcher": [] } ] } \ No newline at end of file From 9c903f1b0107343786498aa47f4027eddb16cc40 Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 27 Oct 2021 20:42:03 +0000 Subject: [PATCH 3/6] misc: improve devcontainer environment setup - Legacy-Id: 19481 --- .devcontainer/devcontainer.json | 5 +++-- .devcontainer/docker-compose.yml | 1 + .devcontainer/init.sh | 4 +++- .vscode/settings.json | 11 ----------- 4 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7574cbc0e..88fa4bccc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,6 +10,7 @@ // Set *default* container specific settings.json values on container create. "settings": { + "terminal.integrated.defaultProfile.linux": "zsh", "python.pythonPath": "/usr/local/bin/python", "python.languageServer": "Pylance", "python.linting.enabled": true, @@ -35,6 +36,7 @@ "-p", "test*.py" ] + // "python.envFile": "${workspaceFolder}/.devcontainer/dev.env" }, // Add the IDs of extensions you want installed when the container is created. @@ -46,11 +48,10 @@ "redhat.vscode-yaml", "visualstudioexptteam.vscodeintellicode", "batisteo.vscode-django", - "littlefoxteam.vscode-python-test-adapter" ], // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [8000, 3306], + "forwardPorts": [8000, 3306] // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "sh /docker-init.sh", diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 1cdb8413f..9661ec999 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -29,6 +29,7 @@ services: UID: 1001 GID: 1001 DATADIR: data + # DJANGO_SETTINGS_MODULE: settings_sqlitetest # Uncomment the next line to use a non-root user for all processes. # user: vscode diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh index 0faed96ab..9c9e4185f 100755 --- a/.devcontainer/init.sh +++ b/.devcontainer/init.sh @@ -75,7 +75,7 @@ if [ ! -f $VIRTDIR/bin/activate ]; then # datatracker virtual python environment on docker container entry: " >> /etc/bash.bashrc cat $VIRTDIR/bin/activate >> /etc/bash.bashrc - cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc + cat /usr/local/share/datatracker/docker/setprompt >> /etc/bash.bashrc else echo "Using virtual environment at $VIRTDIR" fi @@ -151,6 +151,8 @@ fi HOME=/opt/home/$USER +/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local + # su -p $USER exec "$@" \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 3e5e7f8dd..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "python.testing.unittestArgs": [ - "-v", - "-s", - "./ietf", - "-p", - "test*.py" - ], - "python.testing.pytestEnabled": false, - "python.testing.unittestEnabled": true -} \ No newline at end of file From c2979adc2b327d9af51efa058a66f810a3b11881 Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 2 Nov 2021 18:08:04 +0000 Subject: [PATCH 4/6] misc: devcontainer - django debug toolbar profile + dockerfile cleanup - Legacy-Id: 19529 --- .devcontainer/Dockerfile | 16 +------------ .devcontainer/init.sh | 10 ++++++-- .devcontainer/settings_local.py | 6 ++--- .devcontainer/settings_local_debug.py | 33 +++++++++++++++++++++++++++ .vscode/launch.json | 13 +++++++++++ ietf/group/views.py | 3 ++- ietf/urls.py | 8 +++++++ 7 files changed, 68 insertions(+), 21 deletions(-) create mode 100644 .devcontainer/settings_local_debug.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 640444d41..f761d9475 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -64,14 +64,6 @@ RUN apt-get install -qy \ zile \ zlib1g-dev -# Install libyang -# RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key && \ -# apt-key add - < Release.key && \ -# rm Release.key && \ -# echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list && \ -# apt-get update && \ -# apt-get install -qy libyang1 - # Install chromedriver RUN apt-get update && \ apt-get install -y gnupg wget curl unzip --no-install-recommends && \ @@ -120,12 +112,6 @@ RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requ COPY .devcontainer/init.sh /docker-init.sh RUN sed -i 's/\r$//' /docker-init.sh && \ chmod +x /docker-init.sh + # ENTRYPOINT ["/usr/local/share/datatracker/.devcontainer/init.sh"] CMD ["sleep", "infinity"] - -# [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends - -# [Optional] Uncomment this line to install global node packages. -# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh index 9c9e4185f..d29a017bc 100755 --- a/.devcontainer/init.sh +++ b/.devcontainer/init.sh @@ -83,11 +83,16 @@ fi echo "Activating the virtual python environment ..." . $VIRTDIR/bin/activate -if [ ! -f $WORKSPACEDIR/ietf/settings_local.py ]; then +if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then echo "Setting up a default settings_local.py ..." cp $WORKSPACEDIR/.devcontainer/settings_local.py $WORKSPACEDIR/ietf/settings_local.py fi +if [ ! -f "$WORKSPACEDIR/ietf/settings_local_debug.py" ]; then + echo "Setting up a default settings_local_debug.py ..." + cp $WORKSPACEDIR/.devcontainer/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py +fi + for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do dir="$WORKSPACEDIR/$sub" if [ ! -d "$dir" ]; then @@ -141,7 +146,6 @@ chmod -R g+w /usr/local/lib/ # so we can patch libs if needed cd "$WORKSPACEDIR" || cd "/home/$USER/" -echo "Done!" if ! echo "$LANG" | grep "UTF-8"; then echo "" echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc" @@ -153,6 +157,8 @@ HOME=/opt/home/$USER /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local +echo "Done!" + # su -p $USER exec "$@" \ No newline at end of file diff --git a/.devcontainer/settings_local.py b/.devcontainer/settings_local.py index 708ac61f9..ca1b0ce6d 100644 --- a/.devcontainer/settings_local.py +++ b/.devcontainer/settings_local.py @@ -5,13 +5,15 @@ import six if six.PY3: from typing import Collection, Dict, List, Tuple # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore + ALLOWED_HOSTS = ['*'] SECRET_KEY = 'jzv$o93h_lzw4a0%0oz-5t5lk+ai=3f8x@uo*9ahu8w4i300o6' DATABASES = { 'default': { - 'HOST': 'localhost', + 'HOST': 'db', 'PORT': 3306, 'NAME': 'ietf_utf8', 'ENGINE': 'django.db.backends.mysql', @@ -61,5 +63,3 @@ SUBMIT_YANG_DRAFT_MODEL_DIR = 'data/developers/ietf-ftp/yang/draftmod/' SUBMIT_YANG_INVAL_MODEL_DIR = 'data/developers/ietf-ftp/yang/invalmod/' SUBMIT_YANG_IANA_MODEL_DIR = 'data/developers/ietf-ftp/yang/ianamod/' SUBMIT_YANG_RFC_MODEL_DIR = 'data/developers/ietf-ftp/yang/rfcmod/' - - diff --git a/.devcontainer/settings_local_debug.py b/.devcontainer/settings_local_debug.py new file mode 100644 index 000000000..4c8a22ce8 --- /dev/null +++ b/.devcontainer/settings_local_debug.py @@ -0,0 +1,33 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from ietf.settings import INSTALLED_APPS, MIDDLEWARE +from ietf.settings_local import * # pyflakes:ignore + +USE_DEBUG_TOOLBAR = True + +INSTALLED_APPS = INSTALLED_APPS + ['debug_toolbar', ] + +MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware', ] + MIDDLEWARE + +LOGGING = { + 'version': 1, + 'filters': { + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue', + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + 'django.db.backends': { + 'level': 'DEBUG', + 'handlers': ['console'], + } + } +} diff --git a/.vscode/launch.json b/.vscode/launch.json index fb7ccb05a..8176b4395 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,6 +16,19 @@ "--settings=settings_local" ], "django": true + }, + { + "name": "Run Server with Debug Toolbar", + "type": "python", + "request": "launch", + "python": "/usr/local/bin/python", + "program": "${workspaceFolder}/ietf/manage.py", + "args": [ + "runserver", + "0.0.0.0:8000", + "--settings=settings_local_debug" + ], + "django": true } ] } \ No newline at end of file diff --git a/ietf/group/views.py b/ietf/group/views.py index d41a925f3..8c8cd238d 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -39,6 +39,7 @@ import datetime import itertools import io import json +from traceback import print_list import markdown import math import os @@ -63,7 +64,7 @@ import debug # pyflakes:ignore from ietf.community.models import CommunityList, EmailSubscription from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import DocTagName, State, DocAlias, RelatedDocument, Document +from ietf.doc.models import DocTagName, State, DocAlias, RelatedDocument, Document, Person from ietf.doc.templatetags.ietf_filters import clean_whitespace from ietf.doc.utils import get_chartering_type, get_tags_for_stream_id from ietf.doc.utils_charter import charter_name_for_group, replace_charter_of_replaced_group diff --git a/ietf/urls.py b/ietf/urls.py index 91e3920bc..fbf1c1b34 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -9,6 +9,7 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.views import static as static_view from django.views.generic import TemplateView from django.views.defaults import server_error +from django.urls import path import debug # pyflakes:ignore @@ -89,3 +90,10 @@ if settings.SERVER_MODE in ('development', 'test'): urlpatterns += static_url(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) settings.DEBUG = save_debug +# Debug Toolbar +if hasattr(settings, 'USE_DEBUG_TOOLBAR') and settings.USE_DEBUG_TOOLBAR: + try: + import debug_toolbar + path('__debug__/', include(debug_toolbar.urls)), + except ImportError: + pass From 8559024b8bf77e08206b4bdad80e8ac7fd0d606f Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 2 Nov 2021 18:37:54 +0000 Subject: [PATCH 5/6] fix: django-debug-toolbar urls import + missing pip requirement - Legacy-Id: 19534 --- ietf/urls.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/urls.py b/ietf/urls.py index fbf1c1b34..9fa40c1ed 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -94,6 +94,6 @@ if settings.SERVER_MODE in ('development', 'test'): if hasattr(settings, 'USE_DEBUG_TOOLBAR') and settings.USE_DEBUG_TOOLBAR: try: import debug_toolbar - path('__debug__/', include(debug_toolbar.urls)), + urlpatterns = urlpatterns + [path('__debug__/', include(debug_toolbar.urls)), ] except ImportError: pass diff --git a/requirements.txt b/requirements.txt index 177472edd..c26b799d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ Django>=2.2.15,<3.0 django-bootstrap3>=14.0 django-csp>=3.5 django-cors-headers>=2.4.0 +django-debug-toolbar>=3.2.2 django-form-utils>=1.0.3 django-formtools>=1.0 # instead of django.contrib.formtools in 1.8 django-markup>=1.5 From d546ad0392c30be8858744e16d086819df23d2ba Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 2 Nov 2021 18:51:00 +0000 Subject: [PATCH 6/6] misc: revert ietf/group/views.py changes - Legacy-Id: 19535 --- ietf/group/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ietf/group/views.py b/ietf/group/views.py index 8c8cd238d..d41a925f3 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -39,7 +39,6 @@ import datetime import itertools import io import json -from traceback import print_list import markdown import math import os @@ -64,7 +63,7 @@ import debug # pyflakes:ignore from ietf.community.models import CommunityList, EmailSubscription from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import DocTagName, State, DocAlias, RelatedDocument, Document, Person +from ietf.doc.models import DocTagName, State, DocAlias, RelatedDocument, Document from ietf.doc.templatetags.ietf_filters import clean_whitespace from ietf.doc.utils import get_chartering_type, get_tags_for_stream_id from ietf.doc.utils_charter import charter_name_for_group, replace_charter_of_replaced_group