diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index c321b5ee1..53c10c01c 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -8,6 +8,7 @@ services: - .:/workspace - /workspace/.parcel-cache - /workspace/__pycache__ + - app-assets:/assets - datatracker-vscode-ext:/root/.vscode-server/extensions # Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function. network_mode: service:db diff --git a/docker-compose.yml b/docker-compose.yml index 177e36539..9b2deab0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,3 +60,4 @@ services: volumes: mariadb-data: + app-assets: diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index f1116a2c3..91726167c 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -48,3 +48,5 @@ USER dev:dev COPY requirements.txt /tmp/pip-tmp/ RUN pip3 --disable-pip-version-check --no-cache-dir install --user --no-warn-script-location -r /tmp/pip-tmp/requirements.txt RUN sudo rm -rf /tmp/pip-tmp + +VOLUME [ "/assets" ] diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 55599602a..61b168880 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -125,6 +125,9 @@ RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf ADD https://raw.githubusercontent.com/eficode/wait-for/v2.1.3/wait-for /usr/local/bin/ RUN chmod +rx /usr/local/bin/wait-for +# Create assets directory +RUN mkdir -p /assets + # Create workspace RUN mkdir -p /workspace WORKDIR /workspace diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 514c593dd..2b3d54138 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -31,25 +31,25 @@ INTERNET_DRAFT_ARCHIVE_DIR = "test/archive/" INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "test/archive/" RFC_PATH = "test/rfc/" -AGENDA_PATH = 'data/developers/www6s/proceedings/' +AGENDA_PATH = '/assets/www6s/proceedings/' MEETINGHOST_LOGO_PATH = AGENDA_PATH USING_DEBUG_EMAIL_SERVER=True EMAIL_HOST='localhost' EMAIL_PORT=2025 -MEDIA_BASE_DIR = 'data/developers' +MEDIA_BASE_DIR = '/assets' MEDIA_ROOT = MEDIA_BASE_DIR + '/media/' MEDIA_URL = '/media/' PHOTOS_DIRNAME = 'photo' PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME -SUBMIT_YANG_CATALOG_MODEL_DIR = 'data/developers/ietf-ftp/yang/catalogmod/' -SUBMIT_YANG_DRAFT_MODEL_DIR = 'data/developers/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = 'data/developers/ietf-ftp/yang/invalmod/' -SUBMIT_YANG_IANA_MODEL_DIR = 'data/developers/ietf-ftp/yang/ianamod/' -SUBMIT_YANG_RFC_MODEL_DIR = 'data/developers/ietf-ftp/yang/rfcmod/' +SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' +SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' +SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' # Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker import socket @@ -60,15 +60,15 @@ INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] # 'ietf.context_processors.sql_debug', # ] -DOCUMENT_PATH_PATTERN = 'data/developers/ietf-ftp/{doc.type_id}/' -INTERNET_DRAFT_PATH = 'data/developers/ietf-ftp/internet-drafts/' -RFC_PATH = 'data/developers/ietf-ftp/rfc/' -CHARTER_PATH = 'data/developers/ietf-ftp/charter/' -BOFREQ_PATH = 'data/developers/ietf-ftp/bofreq/' -CONFLICT_REVIEW_PATH = 'data/developers/ietf-ftp/conflict-reviews/' -STATUS_CHANGE_PATH = 'data/developers/ietf-ftp/status-changes/' -INTERNET_DRAFT_ARCHIVE_DIR = 'data/developers/ietf-ftp/internet-drafts/' -INTERNET_ALL_DRAFTS_ARCHIVE_DIR = 'data/developers/ietf-ftp/internet-drafts/' +DOCUMENT_PATH_PATTERN = '/assets/ietf-ftp/{doc.type_id}/' +INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/' +RFC_PATH = '/assets/ietf-ftp/rfc/' +CHARTER_PATH = '/assets/ietf-ftp/charter/' +BOFREQ_PATH = '/assets/ietf-ftp/bofreq/' +CONFLICT_REVIEW_PATH = '/assets/ietf-ftp/conflict-reviews/' +STATUS_CHANGE_PATH = '/assets/ietf-ftp/status-changes/' +INTERNET_DRAFT_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' +INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/' NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/' SLIDE_STAGING_PATH = 'test/staging/' diff --git a/docker/configs/settings_local_sqlitetest.py b/docker/configs/settings_local_sqlitetest.py index b852ba4bd..268fe6ec1 100644 --- a/docker/configs/settings_local_sqlitetest.py +++ b/docker/configs/settings_local_sqlitetest.py @@ -56,7 +56,7 @@ INTERNET_DRAFT_ARCHIVE_DIR = "test/archive/" INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "test/archive/" RFC_PATH = "test/rfc/" -AGENDA_PATH = 'data/developers/www6s/proceedings/' +AGENDA_PATH = '/assets/www6s/proceedings/' MEETINGHOST_LOGO_PATH = AGENDA_PATH USING_DEBUG_EMAIL_SERVER=True @@ -70,12 +70,12 @@ MEDIA_URL = '/media/' PHOTOS_DIRNAME = 'photo' PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME -DOCUMENT_PATH_PATTERN = 'data/developers/ietf-ftp/{doc.type_id}/' +DOCUMENT_PATH_PATTERN = '/assets/ietf-ftp/{doc.type_id}/' -SUBMIT_YANG_CATALOG_MODEL_DIR = 'data/developers/ietf-ftp/yang/catalogmod/' -SUBMIT_YANG_DRAFT_MODEL_DIR = 'data/developers/ietf-ftp/yang/draftmod/' -SUBMIT_YANG_INVAL_MODEL_DIR = 'data/developers/ietf-ftp/yang/invalmod/' -SUBMIT_YANG_IANA_MODEL_DIR = 'data/developers/ietf-ftp/yang/ianamod/' -SUBMIT_YANG_RFC_MODEL_DIR = 'data/developers/ietf-ftp/yang/rfcmod/' +SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/' +SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/' +SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/' +SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/' +SUBMIT_YANG_RFC_MODEL_DIR = '/assets/ietf-ftp/yang/rfcmod/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index 36d1a61e7..e3eff6061 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -8,6 +8,7 @@ services: - .:/workspace - /workspace/.parcel-cache - /workspace/__pycache__ + - app-assets:/assets db: ports: - '3306' diff --git a/docker/scripts/app-create-dirs.sh b/docker/scripts/app-create-dirs.sh index 35e230fee..807522e25 100755 --- a/docker/scripts/app-create-dirs.sh +++ b/docker/scripts/app-create-dirs.sh @@ -8,26 +8,26 @@ for sub in \ test/media \ test/wiki/ietf \ data/nomcom_keys/public_keys \ - data/developers/ietf-ftp \ - data/developers/ietf-ftp/bofreq \ - data/developers/ietf-ftp/charter \ - data/developers/ietf-ftp/conflict-reviews \ - data/developers/ietf-ftp/internet-drafts \ - data/developers/ietf-ftp/rfc \ - data/developers/ietf-ftp/status-changes \ - data/developers/ietf-ftp/yang/catalogmod \ - data/developers/ietf-ftp/yang/draftmod \ - data/developers/ietf-ftp/yang/ianamod \ - data/developers/ietf-ftp/yang/invalmod \ - data/developers/ietf-ftp/yang/rfcmod \ - data/developers/www6s \ - data/developers/www6s/staging \ - data/developers/www6s/wg-descriptions \ - data/developers/www6s/proceedings \ - data/developers/www6/ \ - data/developers/www6/iesg \ - data/developers/www6/iesg/evaluation \ - data/developers/media/photo \ + /assets/ietf-ftp \ + /assets/ietf-ftp/bofreq \ + /assets/ietf-ftp/charter \ + /assets/ietf-ftp/conflict-reviews \ + /assets/ietf-ftp/internet-drafts \ + /assets/ietf-ftp/rfc \ + /assets/ietf-ftp/status-changes \ + /assets/ietf-ftp/yang/catalogmod \ + /assets/ietf-ftp/yang/draftmod \ + /assets/ietf-ftp/yang/ianamod \ + /assets/ietf-ftp/yang/invalmod \ + /assets/ietf-ftp/yang/rfcmod \ + /assets/www6s \ + /assets/www6s/staging \ + /assets/www6s/wg-descriptions \ + /assets/www6s/proceedings \ + /assets/www6/ \ + /assets/www6/iesg \ + /assets/www6/iesg/evaluation \ + /assets/media/photo \ ; do if [ ! -d "$sub" ]; then echo "Creating dir $sub" diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index ac2bebd0e..2fbc1e0a4 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -2,12 +2,13 @@ WORKSPACEDIR="/workspace" -sudo service rsyslog start +sudo service rsyslog start &>/dev/null # Fix ownership of volumes echo "Fixing volumes ownership..." sudo chown -R dev:dev "$WORKSPACEDIR/.parcel-cache" sudo chown -R dev:dev "$WORKSPACEDIR/__pycache__" +sudo chown dev:dev "/assets" echo "Fix chromedriver /dev/shm permissions..." sudo chmod 1777 /dev/shm diff --git a/docker/scripts/app-rsync-extras.sh b/docker/scripts/app-rsync-extras.sh index 074826049..aa981a3e6 100755 --- a/docker/scripts/app-rsync-extras.sh +++ b/docker/scripts/app-rsync-extras.sh @@ -1,6 +1,6 @@ #!/bin/bash -version=0.11 +version=0.12 program=${0##*/} progdir=${0%/*} if [ "$progdir" = "$program" ]; then progdir="."; fi @@ -23,7 +23,7 @@ DESCRIPTION from the ietf server to a local directory, for instance drafts, charters, rfcs, agendas, minutes, etc. - If no destination is given, the default is data/developers. + If no destination is given, the default is /assets. OPTIONS EOF @@ -32,7 +32,7 @@ EOF AUTHOR Written by: - Henrik Levkowetz, + Henrik Levkowetz, Lars Eggert, COPYRIGHT @@ -40,7 +40,7 @@ COPYRIGHT the code. All rights reserved. Redistribution and use in source and binary forms, with or without modification, is permitted pursuant to, and subject to the license terms contained in, the Revised BSD - License set forth in Section 4.c of the IETF Trust’s Legal Provisions + License set forth in Section 4.c of the IETF Trust's Legal Provisions Relating to IETF Documents(https://trustee.ietf.org/license-info). EOF @@ -61,52 +61,51 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; # Option parsing -shortopts=hvV +shortopts=hv args=$(getopt -o$shortopts $*) if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi set -- $args while true ; do case "$1" in - -h) usage; exit;; # Show this help, then exit - -v) VERBOSE=1;; # Be more talkative - -V) version; exit;; # Show program version, then exit - --) shift; break;; - *) die "Internal error, inconsistent option specification: '$1'";; + -h) usage; exit;; # Show this help, then exit + -v) version; exit;; # Show program version, then exit + --) shift; break;; + *) die "Internal error, inconsistent option specification: '$1'";; esac shift done # The program itself if [ $# -lt 1 ]; then - DEST_ROOT=data/developers + DEST_ROOT=/assets else DEST_ROOT="${1%/}" fi echo "Using destination $DEST_ROOT" for dir in bofreq; do - dest="$DEST_ROOT/ietf-ftp/$dir" - mkdir -p "$dest" - echo "Fetching $dest ..." - rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::$dir/ $dest/ + dest="$DEST_ROOT/ietf-ftp/$dir" + mkdir -p "$dest" + echo "Fetching $dest ..." + rsync -auz --info=progress2 rsync.ietf.org::$dir/ $dest/ done for dir in charter conflict-reviews internet-drafts review rfc slides status-changes yang; do - dest="$DEST_ROOT/ietf-ftp/$dir" - mkdir -p "$dest" - echo "Fetching $dest ..." - rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::everything-ftp/$dir/ $dest/ + dest="$DEST_ROOT/ietf-ftp/$dir" + mkdir -p "$dest" + echo "Fetching $dest ..." + rsync -auz --info=progress2 rsync.ietf.org::everything-ftp/$dir/ $dest/ done for dir in floor photo; do - dest="$DEST_ROOT/media/$dir" - mkdir -p "$dest" - echo "Fetching $dest ..." - rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::dev.media/$dir/ $dest/ + dest="$DEST_ROOT/media/$dir" + mkdir -p "$dest" + echo "Fetching $dest ..." + rsync -auz --info=progress2 rsync.ietf.org::dev.media/$dir/ $dest/ done dest="$DEST_ROOT/archive/id" mkdir -p "$dest" echo "Fetching $dest ..." -rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::id-archive/ $dest/ +rsync -auz --info=progress2 rsync.ietf.org::id-archive/ $dest/