diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..f761d9475 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,117 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT="3.10-bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +EXPOSE 8000 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update +RUN apt-get -qy upgrade + +# Install the packages we need +RUN apt-get install -qy \ + bash \ + build-essential \ + bzip2 \ + ca-certificates \ + colordiff \ + gawk \ + gcc \ + python3-ipython \ + jq \ + less \ + libbz2-dev \ + libdb5.3-dev \ + libexpat1-dev \ + libffi-dev \ + libgdbm-dev \ + libjpeg62-turbo-dev \ + liblzma-dev \ + libmagic1 \ + libmariadb-dev-compat \ + libmariadb-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libsvn1 \ + libxml2-dev \ + libxslt-dev \ + libyang1 \ + libz-dev \ + libffi-dev \ + locales \ + make \ + man \ + mariadb-client \ + netcat \ + openssh-client \ + patch \ + procps \ + pv \ + rsync \ + rsyslog \ + subversion \ + sudo \ + uuid-dev \ + vim \ + wget \ + xz-utils\ + zile \ + zlib1g-dev + +# Install chromedriver +RUN apt-get update && \ + apt-get install -y gnupg wget curl unzip --no-install-recommends && \ + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \ + apt-get update -y && \ + apt-get install -y google-chrome-stable && \ + CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \ + DRIVERVER=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") && \ + wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" && \ + unzip /chromedriver/chromedriver* -d /chromedriver && \ + ln -s /chromedriver/chromedriver /usr/local/bin/chromedriver && \ + ln -s /chromedriver/chromedriver /usr/bin/chromedriver + +# Get rid of installation files we don't need in the image, to reduce size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Enable some common locales +RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen && \ + locale-gen + +# Remove an rsyslog module that we don't need, which also requires extra permissions +RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf + +# Set up root password +RUN echo "root:root" | chpasswd + +# idnits and dependencies +ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/ +RUN chmod +rx /usr/local/bin/idnits + +# Directory for Mac certs +RUN mkdir /etc/certificates + +# Setup workspace +ENV HOSTNAME="datatracker" + +ENV DDIR="/usr/local/share/datatracker" +RUN mkdir -p $DDIR +WORKDIR $DDIR + +COPY requirements.txt /tmp/pip-tmp/ +RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ + && rm -rf /tmp/pip-tmp + +COPY .devcontainer/init.sh /docker-init.sh +RUN sed -i 's/\r$//' /docker-init.sh && \ + chmod +x /docker-init.sh + +# ENTRYPOINT ["/usr/local/share/datatracker/.devcontainer/init.sh"] +CMD ["sleep", "infinity"] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..88fa4bccc --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,61 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.202.5/containers/python-3 +{ + "name": "IETF Datatracker", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/usr/local/share/datatracker", + "shutdownAction": "stopCompose", + "postCreateCommand": "/docker-init.sh", + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.defaultProfile.linux": "zsh", + "python.pythonPath": "/usr/local/bin/python", + "python.languageServer": "Pylance", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestArgs": [ + "ietf" + ], + "python.testing.unittestEnabled": true, + "python.testing.pytestEnabled": false, + "python.testing.unittestArgs": [ + "-v", + "-s", + "./ietf", + "-p", + "test*.py" + ] + // "python.envFile": "${workspaceFolder}/.devcontainer/dev.env" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-azuretools.vscode-docker", + "editorconfig.editorconfig", + "redhat.vscode-yaml", + "visualstudioexptteam.vscodeintellicode", + "batisteo.vscode-django", + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [8000, 3306] + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "sh /docker-init.sh", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "django" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..9661ec999 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,57 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile + args: + # Update 'VARIANT' to pick a version of Python: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + # Append -bullseye or -buster to pin to an OS version. + # Use -bullseye variants on local arm64/Apple Silicon. + VARIANT: 3.6-bullseye + + volumes: + - ..:/usr/local/share/datatracker:cached + init: true + + # Overrides default command so things don't shut down after the process ends. + command: sleep infinity + + # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. + network_mode: service:db + + depends_on: + - db + + environment: + USER: django + UID: 1001 + GID: 1001 + DATADIR: data + # DJANGO_SETTINGS_MODULE: settings_sqlitetest + + # Uncomment the next line to use a non-root user for all processes. + # user: vscode + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + db: + image: mariadb:10 + restart: unless-stopped + volumes: + - mariadb-data:/var/lib/mysql + environment: + MYSQL_ROOT_PASSWORD: ietf + MYSQL_DATABASE: ietf_utf8 + MYSQL_USER: django + MYSQL_PASSWORD: RkTkDPFnKpko + command: ['--character-set-server=utf8mb4', '--collation-server=utf8mb4_unicode_ci'] + + + # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + +volumes: + mariadb-data: \ No newline at end of file diff --git a/.devcontainer/init.sh b/.devcontainer/init.sh new file mode 100755 index 000000000..d29a017bc --- /dev/null +++ b/.devcontainer/init.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# A little bit of setup +export LANG=en_GB.UTF-8 + +WORKSPACEDIR="/usr/local/share/datatracker" + +echo "Gathering info ..." +if [ ! "$USER" ]; then + echo "Environment variable USER is not set -- will set USER='django'." + USER="django" +fi +if [ ! "$UID" ]; then + echo "Environment variable UID is not set -- will set UID='1000'." + UID="1000" +fi +if [ ! "$GID" ]; then + echo "Environment variable GID is not set -- will set GID='1000'." + GID="1000" +fi +if [ ! "$TAG" ]; then + echo "Environment variable TAG is not set -- will set TAG='datatracker'." + TAG="datatracker" +fi +echo "User $USER ($UID:$GID)" + +echo "Checking if syslogd is running ..." +if ! /etc/init.d/rsyslog status > /dev/null; then + echo "Starting syslogd ..." + /etc/init.d/rsyslog start +fi + +echo "Waiting for DB container to come online ..." +wget -qO- https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for | sh -s -- localhost:3306 -- echo "DB ready" + +echo "Checking if the IETF database exists in DB container ..." +if ! mysql --protocol tcp -h localhost -u root --password=ietf --database="ietf_utf8" --execute="SHOW TABLES;" | grep -q 'django'; then + echo "Fetching database ..." + DUMPDIR=/home/$USER/$DATADIR + wget -N --progress=bar:force:noscroll -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz + + echo "Creating database ..." + mysql --protocol tcp -h localhost -u root --password=ietf --database="ietf_utf8" --execute="DROP DATABASE IF EXISTS ietf_utf8;" + mysqladmin --protocol tcp -h localhost -u root --password=ietf --default-character-set=utf8 create ietf_utf8 + + echo "Setting up permissions ..." + mysql --protocol tcp -h localhost -u root --password="ietf" ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO 'django'@'%' IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;" + + echo "Loading database (this may take a while)..." + gunzip < $DUMPDIR/ietf_utf8.sql.gz \ + | pv --progress --bytes --rate --eta --cursor --force --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \ + | sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \ + | mysql --protocol tcp -h localhost -u django --password=RkTkDPFnKpko -s -f ietf_utf8 \ + && rm $DUMPDIR/ietf_utf8.sql.gz +fi + +if ! grep -q ":$GID:$" /etc/group ; then + echo "Creating group entry for GID '$GID' ..." + groupadd -g "$GID" "$USER" +fi +if ! id -u "$USER" &> /dev/null; then + echo "Creating user '$USER' ..." + useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER + echo "$USER:$USER" | chpasswd +fi + +VIRTDIR="/opt/home/$USER/$TAG" +echo "Checking that there's a virtual environment for $TAG ..." +if [ ! -f $VIRTDIR/bin/activate ]; then + echo "Setting up python virtualenv at $VIRTDIR ..." + mkdir -p $VIRTDIR + python3.6 -m venv $VIRTDIR + echo -e " +# This is from $VIRTDIR/bin/activate, to activate the +# datatracker virtual python environment on docker container entry: +" >> /etc/bash.bashrc + cat $VIRTDIR/bin/activate >> /etc/bash.bashrc + cat /usr/local/share/datatracker/docker/setprompt >> /etc/bash.bashrc +else + echo "Using virtual environment at $VIRTDIR" +fi + +echo "Activating the virtual python environment ..." +. $VIRTDIR/bin/activate + +if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then + echo "Setting up a default settings_local.py ..." + cp $WORKSPACEDIR/.devcontainer/settings_local.py $WORKSPACEDIR/ietf/settings_local.py +fi + +if [ ! -f "$WORKSPACEDIR/ietf/settings_local_debug.py" ]; then + echo "Setting up a default settings_local_debug.py ..." + cp $WORKSPACEDIR/.devcontainer/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py +fi + +for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do + dir="$WORKSPACEDIR/$sub" + if [ ! -d "$dir" ]; then + echo "Creating dir $dir" + mkdir -p "$dir"; + fi +done + +for sub in \ + nomcom_keys/public_keys \ + developers/ietf-ftp \ + developers/ietf-ftp/bofreq \ + developers/ietf-ftp/charter \ + developers/ietf-ftp/conflict-reviews \ + developers/ietf-ftp/internet-drafts \ + developers/ietf-ftp/rfc \ + developers/ietf-ftp/status-changes \ + developers/ietf-ftp/yang/catalogmod \ + developers/ietf-ftp/yang/draftmod \ + developers/ietf-ftp/yang/ianamod \ + developers/ietf-ftp/yang/invalmod \ + developers/ietf-ftp/yang/rfcmod \ + developers/www6s \ + developers/www6s/staging \ + developers/www6s/wg-descriptions \ + developers/www6s/proceedings \ + developers/www6/ \ + developers/www6/iesg \ + developers/www6/iesg/evaluation \ + ; do + dir="$WORKSPACEDIR/data/$sub" + if [ ! -d "$dir" ]; then + echo "Creating dir $dir" + mkdir -p "$dir"; + chown "$USER" "$dir" + fi +done + +if [ ! -f "$WORKSPACEDIR/test/data/draft-aliases" ]; then + echo "Generating draft aliases ..." + ietf/bin/generate-draft-aliases } +fi + +if [ ! -f "$WORKSPACEDIR/test/data/group-aliases" ]; then + echo "Generating group aliases ..." + ietf/bin/generate-wg-aliases } +fi + +chown -R $USER /opt/home/$USER +chmod -R g+w /usr/local/lib/ # so we can patch libs if needed + +cd "$WORKSPACEDIR" || cd "/home/$USER/" + +if ! echo "$LANG" | grep "UTF-8"; then + echo "" + echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc" +else + echo "LANG=$LANG" +fi + +HOME=/opt/home/$USER + +/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local + +echo "Done!" + +# su -p $USER + +exec "$@" \ No newline at end of file diff --git a/.devcontainer/settings_local.py b/.devcontainer/settings_local.py new file mode 100644 index 000000000..ca1b0ce6d --- /dev/null +++ b/.devcontainer/settings_local.py @@ -0,0 +1,65 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +import six +if six.PY3: + from typing import Collection, Dict, List, Tuple # pyflakes:ignore + +from ietf.settings import * # pyflakes:ignore + +ALLOWED_HOSTS = ['*'] + +SECRET_KEY = 'jzv$o93h_lzw4a0%0oz-5t5lk+ai=3f8x@uo*9ahu8w4i300o6' + +DATABASES = { + 'default': { + 'HOST': 'db', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +} # type: Dict[str, Dict[str, Collection[str]]] + +DATABASE_TEST_OPTIONS = { + 'init_command': 'SET storage_engine=InnoDB', + } + +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" +IDSUBMIT_REPOSITORY_PATH = "test/id/" +IDSUBMIT_STAGING_PATH = "test/staging/" +INTERNET_DRAFT_ARCHIVE_DIR = "test/archive/" +INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "test/archive/" +RFC_PATH = "test/rfc/" + +AGENDA_PATH = 'data/developers/www6s/proceedings/' +MEETINGHOST_LOGO_PATH = AGENDA_PATH + +USING_DEBUG_EMAIL_SERVER=True +EMAIL_HOST='localhost' +EMAIL_PORT=2025 + +TRAC_WIKI_DIR_PATTERN = "test/wiki/%s" +TRAC_SVN_DIR_PATTERN = "test/svn/%s" +TRAC_CREATE_ADHOC_WIKIS = [ +] # type: List[Tuple[str, str, str]] + +MEDIA_BASE_DIR = 'test' +MEDIA_ROOT = MEDIA_BASE_DIR + '/media/' +MEDIA_URL = '/media/' + +PHOTOS_DIRNAME = 'photo' +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +DOCUMENT_PATH_PATTERN = 'data/developers/ietf-ftp/{doc.type_id}/' + +SUBMIT_YANG_CATALOG_MODEL_DIR = 'data/developers/ietf-ftp/yang/catalogmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = 'data/developers/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = 'data/developers/ietf-ftp/yang/invalmod/' +SUBMIT_YANG_IANA_MODEL_DIR = 'data/developers/ietf-ftp/yang/ianamod/' +SUBMIT_YANG_RFC_MODEL_DIR = 'data/developers/ietf-ftp/yang/rfcmod/' diff --git a/.devcontainer/settings_local_debug.py b/.devcontainer/settings_local_debug.py new file mode 100644 index 000000000..4c8a22ce8 --- /dev/null +++ b/.devcontainer/settings_local_debug.py @@ -0,0 +1,33 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from ietf.settings import INSTALLED_APPS, MIDDLEWARE +from ietf.settings_local import * # pyflakes:ignore + +USE_DEBUG_TOOLBAR = True + +INSTALLED_APPS = INSTALLED_APPS + ['debug_toolbar', ] + +MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware', ] + MIDDLEWARE + +LOGGING = { + 'version': 1, + 'filters': { + 'require_debug_true': { + '()': 'django.utils.log.RequireDebugTrue', + } + }, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'filters': ['require_debug_true'], + 'class': 'logging.StreamHandler', + } + }, + 'loggers': { + 'django.db.backends': { + 'level': 'DEBUG', + 'handlers': ['console'], + } + } +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..8176b4395 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,34 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Run Server", + "type": "python", + "request": "launch", + "python": "/usr/local/bin/python", + "program": "${workspaceFolder}/ietf/manage.py", + "args": [ + "runserver", + "0.0.0.0:8000", + "--settings=settings_local" + ], + "django": true + }, + { + "name": "Run Server with Debug Toolbar", + "type": "python", + "request": "launch", + "python": "/usr/local/bin/python", + "program": "${workspaceFolder}/ietf/manage.py", + "args": [ + "runserver", + "0.0.0.0:8000", + "--settings=settings_local_debug" + ], + "django": true + } + ] +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..1319ef547 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,63 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "Run Checks", + "type": "shell", + "command": "/usr/local/bin/python", + "args": [ + "${workspaceFolder}/ietf/manage.py", + "check", + "--settings=settings_local" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + }, + { + "label": "Run Tests", + "type": "shell", + "command": "/usr/local/bin/python", + "args": [ + "${workspaceFolder}/ietf/manage.py", + "test", + "--settings=settings_sqlitetest" + ], + "group": "test", + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "new", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + }, + { + "label": "Re-run Setup Project", + "type": "shell", + "command": "/bin/bash", + "args": [ + "/docker-init.sh" + ], + "presentation": { + "echo": true, + "reveal": "always", + "focus": true, + "panel": "new", + "showReuseMessage": false, + "clear": false + }, + "problemMatcher": [] + } + ] +} \ No newline at end of file diff --git a/ietf/urls.py b/ietf/urls.py index 91e3920bc..9fa40c1ed 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -9,6 +9,7 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.views import static as static_view from django.views.generic import TemplateView from django.views.defaults import server_error +from django.urls import path import debug # pyflakes:ignore @@ -89,3 +90,10 @@ if settings.SERVER_MODE in ('development', 'test'): urlpatterns += static_url(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) settings.DEBUG = save_debug +# Debug Toolbar +if hasattr(settings, 'USE_DEBUG_TOOLBAR') and settings.USE_DEBUG_TOOLBAR: + try: + import debug_toolbar + urlpatterns = urlpatterns + [path('__debug__/', include(debug_toolbar.urls)), ] + except ImportError: + pass diff --git a/requirements.txt b/requirements.txt index 9952c927d..1e634d153 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ Django>=2.2.15,<3.0 django-bootstrap3>=14.0 django-csp>=3.5 django-cors-headers>=2.4.0 +django-debug-toolbar>=3.2.2 django-form-utils>=1.0.3 django-formtools>=1.0 # instead of django.contrib.formtools in 1.8 django-markup>=1.5