altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Version of the new docker setup emailed out for testing on 2021-10-12

Browse source 
- Legacy-Id: 19417
This commit is contained in:
Lars Eggert 2021-10-13 11:33:37 +00:00
parent 3386e59a61
commit 6fb64998ce
12 changed files with 211 additions and 876 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

226
docker/Dockerfile
View file

@ -1,185 +1,85 @@
# -*- shell-mode -*- # This is a Dockerfile with everything in it to run the IETF datatracker.
# This file is a docker (https://www.docker.com/what-docker) recipe, which can be used to build
# a docker image which is ready to run a datatracker in development mode.
# #
# It is used to build an image (once you've installed docker) using a command like this (assuming # If you make changes to the datatracker that add new dependencies (python
# suitable replacement of $variables: # packages or otherwise), you need to rebuild this image to make them
# available. Do this in the top-level directory of your datatracker source
# tree:
# #
# $ docker build -t $yourdockerid/datatracker:$version # docker build --tag dt:latest .
# #
# To use a pre-built image, assuming we're on OS X and have a checked-out datatracker repository # You can then execute the datatracker like this (also from the top-level
# at /Users/$login/src/6.8.1.dev0, you would start (again assuming you've installed docker) # datatracker source directory):
# a container from an image, as follows: #
# # docker run -ti -v $(pwd):/root/src dt:latest
# $ docker run -ti --name=$containername -v /Users/$login:/home/$login levkowetz/datatracker:$version /bin/bash
#
# This maps your home directory to /home/$login in the container, and starts it running /bin/bash.
#
# In this first version, the docker environment is set up so that tests will run successfully,
# but the database has *not* been loaded with a dump, and supporting files (drafts, charters, etc.)
# have *not* been downloaded.
FROM dyne/devuan:beowulf FROM ubuntu:hirsute
LABEL maintainer="Henrik Levkowetz <henrik@levkowetz.com>" LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
# Default django runserver port # Default django runserver port
EXPOSE 8000 EXPOSE 8000
# Run apt-get noninteractive # Default mysqld/mariadb port
EXPOSE 3306
# Install all dependencies that are available as packages
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
ENV DEVUAN_FRONTEND=noninteractive RUN apt-get -y update && \
apt-get -y install --no-install-recommends \
# Uncomment this to be able to install and run apt-show-versions: apache2-utils \
RUN rm -v /etc/apt/apt.conf.d/docker-compress apt-file \
RUN rm -v /var/lib/apt/lists/*lz4 apt-utils \
chromium-driver \
RUN apt-get update --allow-releaseinfo-change curl \
RUN apt-get install -qy apt-transport-https enscript \
gcc \
# Use backports, updates, and security updates ghostscript \
RUN echo "deb http://deb.devuan.org/merged beowulf main contrib non-free" > /etc/apt/sources.list git \
RUN echo "deb http://deb.devuan.org/merged beowulf-security main contrib non-free" >> /etc/apt/sources.list graphviz \
RUN echo "deb http://deb.devuan.org/merged beowulf-updates main contrib non-free" >> /etc/apt/sources.list libmagic-dev \
RUN echo "deb http://deb.devuan.org/merged beowulf-backports main contrib non-free" >> /etc/apt/sources.list libmariadb-dev \
mariadb-server \
# Remove some excludes for the docker image npm \
RUN sed -i -e '/^path-exclude=.*\/groff/d' \ pigz \
-e '/^path-exclude=.*\/locale/d' \ pv \
-e '/^path-exclude=.*\/man/d' /etc/dpkg/dpkg.cfg.d/docker-excludes python-is-python3 \
python3-dev \
# Install needed packages python3-pip \
#
# We're not including graphviz and ghostscript, needed for the 3 document
# dependency graph tests; they would increase the size of the image by about
# 15%, about 100MB.
# Fetch apt package information, and upgrade to latest package versions
RUN apt-get update
RUN apt-get -qy upgrade
# Install the packages we need
RUN apt-get install -qy \
build-essential \
bzip2 \
ca-certificates \
colordiff \
gawk \
gcc \
ipython \
jq \
less \
libbz2-dev \
libdb5.3-dev \
libexpat1-dev \
libffi-dev \
libgdbm-dev \
libjpeg62-turbo-dev \
liblzma-dev \
libmagic1 \
libmariadbclient-dev \
libncurses5-dev \
libncursesw5-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libsvn1 \
libxml2-dev \
libxslt-dev \
libz-dev \
libffi-dev \
locales \
make \
man \
mariadb-client \
mariadb-server \
openssh-client \
patch \
procps \
pv \
rsync \
rsyslog \ rsyslog \
subversion \ unzip \
sudo \ yang-tools && \
uuid-dev \ apt-get -y clean && \
vim \ rm -rf /var/lib/apt/lists/*
wget \
xz-utils\
zile \
zlib1g-dev
# Postgresql packages # Install bower
RUN apt-get install -qy \ RUN npm install -g bower
postgresql-11 \
postgresql-server-dev-11
# Get the key used to sign the libyang repo # Install idnits
RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key
RUN apt-key add - < Release.key
RUN rm Release.key
# Add apt source entry for libyang
RUN echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list
# Update the package defs, and install the desired mysql from the mysql repo
RUN apt-get update
RUN apt-get install -qy libyang1
# This is expected to exist by the mysql startup scripts:
#RUN touch /etc/mysql/debian.cnf
# ------------------------------------------------------------------------------
# Get rid of installation files we don't need in the image, to reduce size
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
# Enable some common locales
RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen
RUN locale-gen
# Remove an rsyslog module that we don't need, which also requires extra permissions
RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf
# Set up root password
RUN echo "root:root" | chpasswd
# MySQL
VOLUME /var/lib/mysql
# idnits and dependencies
ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/ ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/
RUN chmod +rx /usr/local/bin/idnits RUN chmod +rx /usr/local/bin/idnits
# Directory for Mac certs # Install current datatracker python dependencies
RUN mkdir /etc/certificates COPY requirements.txt /
RUN pip install -r /requirements.txt
# # Python 3 # Turn off rsyslog kernel logging (doesn't work in Docker)
# # Comment in if OS does not provide python3.6, which is the current RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
# # production version
ENV PYVER=3.6.10
ENV PYREV=3.6
WORKDIR /usr/src # Allow access to mariadb over the network
RUN wget -q https://www.python.org/ftp/python/$PYVER/Python-$PYVER.tar.xz RUN sed -i 's/127.0.0.1/0.0.0.0/' /etc/mysql/mariadb.conf.d/50-server.cnf
RUN tar xJf Python-$PYVER.tar.xz
RUN rm Python-$PYVER.tar.xz
WORKDIR /usr/src/Python-$PYVER/
RUN ./configure
RUN make
RUN make altinstall
WORKDIR /usr/src
RUN rm -rf /usr/src/Python-$PYVER/
ENV HOSTNAME="datatracker" # Turn on mariadb performance_schema
RUN sed -i 's/\[mysqld\]/\[mysqld\]\nperformance_schema=ON/' /etc/mysql/mariadb.conf.d/50-server.cnf
ENV DDIR="/usr/local/share/datatracker" # Make the mariadb sys schema available for possible installation
RUN mkdir -p $DDIR # We would normally use the next line, but that has a bug:
WORKDIR $DDIR # ADD https://github.com/FromDual/mariadb-sys/archive/master.zip /
# This is the repo that has the PR:
COPY requirements.txt ./ ADD https://github.com/grooverdan/mariadb-sys/archive/refs/heads/master.zip /
COPY setprompt ./ RUN unzip /master.zip
# Copy the startup file
COPY docker-init.sh /docker-init.sh COPY docker-init.sh /docker-init.sh
RUN chmod +x /docker-init.sh RUN chmod +x /docker-init.sh
ENTRYPOINT ["/docker-init.sh"]
CMD /bin/bash WORKDIR /root/src
ENTRYPOINT ["/docker-init.sh"]

37
docker/README.md Normal file
View file

@ -0,0 +1,37 @@
# Datatracker Development in Docker
1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred
platform.
2. If you have a copy of the datatracker code checked out already, simply `cd`
to the top-level directory.
If not, check out a datatracker branch as usual. We'll check out `trunk`
below, but you can use any branch:
svn co https://svn.ietf.org/svn/tools/ietfdb/trunk
cd trunk
3. **TEMPORARY:** Replace the contents of the `docker` directory with Lars'
files.
4. **TEMPORARY:** Until Lars' changes have been merged and a docker image is
available for download, you will need to build it locally:
docker/build -l
This will take a while, but only needs to be done once.
5. Use the `docker/run` script to start the datatracker container. You will be
dropped into a shell from which you can start the datatracker and execute
related commands as usual, for example:
ietf/manage.py runserver 0.0.0.0:8000
If you do not already have a copy of the IETF database available in the
`data` directory, one will be downloaded and imported the first time you run
`docker/run`. This will take some time.
Once the datatracker has started, you should be able to open
[http://localhost:8000](http://localhost:8000) in a browser and see the
landing page.

98
docker/README.rst
View file

@ -1,98 +0,0 @@
==============================================================================
Datatracker Development in a Docker Container (beta)
==============================================================================
Intro
=====
Docker_ is a toolkit which lets you package software together with its
dependencies in lightweight containers, and run it in isolated virtual
environments.
During and just after IETF-94 I've spent quite a bit of time setting up a
docker image which provides the dependencies needed to run the datatracker,
and it's now available for beta testing. Hopefully this should make it
substantially easier to get started with datatracker development.
Steps
=====
1. Set up Docker on your preferred platform. Official installers exist for
many Linux flavours, OS X, Windows and Cloud services. Here's the full `List
of Installation Instructions`_.
Docker containers require the services of an underlying Linux API, which
means that on OS X and Windows, these have to be provided by a virtual
machine which runs a minimal Linux image. The virtual machine used on
non-Linux platforms is commonly VirtualBox. On Linux kernels with version
3.8 or later, no virtual machine is needed, as the docker images can be
fully supported with the native kernel services.
Please follow the Docker installations all the way through to successfully
running the ``hello-world`` example in a terminal window ( ``$ docker run
hello-world``).
2. Check out your datatracker branch as usual, in a suitable directory.
We'll assume ``~/src/dt/`` here, and assume you are ``'coder'``::
~/src/dt/ $ svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/coder/6.8.2.dev0
3. In the checked-out working copy, you'll find a ``docker/`` directory and a
``data/`` directory at the top level. We're first going to set up a copy of
the MySQL database files under the ``data/`` directory.
There is a command in the ``docker/`` directory, ``setupdb`` which will do
this for you, or you can do it manually.
Either run::
~/src/dt/6.8.2.dev0/ $ docker/setupdb
or do this step-by-step: fetch down a pre-built copy of the datatracker
database, place it in the ``data`` directory, unpack it, and fix
permissions::
~/src/dt/6.8.2.dev0/ $ cd data
~/src/dt/6.8.2.dev0/data/ $ wget https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
~/src/dt/6.8.2.dev0/data/ $ tar xjf ietf_utf8.bin.tar.bz2
~/src/dt/6.8.2.dev0/data/ $ chmod -R go+rwX mysql
4. In the ``docker/`` directory you'll also find a wrapper script named
``'run'``. We will be using the wrapper to run a pre-built docker image
fetched from the docker hub::
~/src/dt/6.8.2.dev0/ $ docker/run
This will pull down the latest docker ietf/datatracker-environment image,
start it up with appropriate settings, map the internal ``/var/lib/mysql/``
directory to the external ``data/mysql/`` directory where we placed the
database, set up a python virtualenv for you, install some dependencies,
and drop you in a bash shell where you can run the datatracker.
6. You are now ready to run the tests::
(virtual) $ ietf/manage.py test --settings=settings_sqlitetest
and then start the dev server::
(virtual) $ ietf/manage.py runserver 0.0.0.0:8000
Note the IP address ``0.0.0.0`` used to make the dev server bind to all
addresses. The internal port 8000 has been mapped to port 8000 externally,
too. In order to find the IP address of the VirtualBox, run ``'$
docker-machine ip'`` *outside* the virtual environment::
~/src/dt/6.8.2.dev0/ $ docker-machine ip
192.168.59.103
~/src/dt/6.8.2.dev0/ $ open http://192.168.59.103:8000/
.. _Docker: https://www.docker.com/
.. _`List of Installation Instructions`: https://docs.docker.com/v1.8/installation/
.. _VirtualBox: https://www.virtualbox.org/

39
docker/build
View file

@ -1,11 +1,11 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.11
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
export LANG=C export LANG=C
@ -31,10 +31,10 @@ DESCRIPTION
EOF EOF
echo -e "OPTIONS" echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ] || [ "$(uname)" = "Darwin" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi fi
cat <<EOF cat <<EOF
@ -48,7 +48,7 @@ COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of Copyright (c) 2016 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. License 'Simplified BSD', as specified
in http://opensource.org/licenses/BSD-3-Clause. in http://opensource.org/licenses/BSD-3-Clause.
EOF EOF
} }
@ -59,10 +59,6 @@ function die() {
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
@ -77,24 +73,22 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
# Options # Options
shortopts=hlt:vV shortopts=hlt:vV
longopts=help,local,tag=,verbose,version longopts=help,local,tag=,version
# Default values # Default values
BRANCH=$(svn log -v ^/tags -l 2 | grep 'A /tags/[1-9]' | awk '{print $2}') IMAGE=ietf/datatracker-environment
TAG=${BRANCH##*/} TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}')")
LOCAL="" LOCAL=""
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args" eval set -- "$args"
sed="sed -r"
else else
# Darwin, BSDs # Darwin, BSDs
args=$(getopt -o$shortopts $SV $*) args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args set -- $args
sed="sed -E"
fi fi
while true ; do while true ; do
@ -102,7 +96,6 @@ while true ; do
-h| --help) usage; exit;; # Show this help, then exit -h| --help) usage; exit;; # Show this help, then exit
-l| --local) LOCAL=1;; # Don't upload -l| --local) LOCAL=1;; # Don't upload
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the latest svn tags name -t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the latest svn tags name
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit -V| --version) version; exit;; # Show program version, then exit
--) shift; break;; --) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";; *) die "Internal error, inconsistent option specification: '$1'";;
@ -113,14 +106,10 @@ done
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# The program itself # The program itself
if [ "$(uname)" == "Linux" ]; then docker rmi -f $IMAGE:trunk 2>/dev/null || true
die "Didn't expect to run this script on Linux -- are you inside docker?" docker build --progress plain -t "$IMAGE:$TAG" docker/
fi docker tag "$(docker images -q $IMAGE | head -n 1)" $IMAGE:latest
docker rmi -f ietf/datatracker-environment:trunk || true
docker build -t ietf/datatracker-environment:$TAG docker/
docker tag $(docker images -q ietf/datatracker-environment | head -n 1) ietf/datatracker-environment:latest
if [ -z "$LOCAL" ]; then if [ -z "$LOCAL" ]; then
docker push ietf/datatracker-environment:latest docker push $IMAGE:latest
docker push ietf/datatracker-environment:$TAG docker push "$IMAGE:$TAG"
fi fi

15
docker/devserver
View file

@ -1,15 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
$progdir/run ietf/manage.py runserver 0.0.0.0:8000

228
docker/docker-init.sh
View file

@ -1,195 +1,85 @@
#!/bin/bash #!/bin/bash
# A little bit of setup MYSQLDIR=/var/lib/mysql
export LANG=en_GB.UTF-8
echo "Gathering info ..." if [ ! -d "$MYSQLDIR" ]; then
MYSQLDIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" echo "WARNING: Expected the directory $MYSQLDIR to exist."
if [ ! "$USER" ]; then exit 1
echo "Environment variable USER is not set -- will set USER='django'."
USER="django"
fi
if [ ! "$UID" ]; then
echo "Environment variable UID is not set -- will set UID='1000'."
UID="1000"
fi
if [ ! "$GID" ]; then
echo "Environment variable GID is not set -- will set GID='1000'."
GID="1000"
fi
if [ ! "$TAG" ]; then
echo "Environment variable TAG is not set -- will set TAG='datatracker'."
TAG="datatracker"
fi
echo "User $USER ($UID:$GID)"
echo "Checking if MySQL base data exists ..."
if [ ! -d $MYSQLDIR/mysql ]; then
echo "WARNING: Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
fi fi
echo "Setting up the 'mysql' user for database file access ..." service rsyslog start
MYSQL_TARGET_GID=$(stat -c "%g" $MYSQLDIR/mysql)
if ! grep -q ":$MYSQL_TARGET_GID:$" /etc/group; then
groupadd -g $MYSQL_TARGET_GID mysqldata
fi
usermod -a -G $MYSQL_TARGET_GID mysql
echo "Checking if MySQL is running ..." if [ -z "$(ls -A $MYSQLDIR/mysql 2>/dev/null)" ]; then
if ! /etc/init.d/mysql status; then echo "WARNING: Database seems to be empty."
echo "Starting mysql ..." mysql_install_db > /dev/null || exit 1
/etc/init.d/mysql start
fi fi
echo "Checking if syslogd is running ..." service mariadb start
if ! /etc/init.d/rsyslog status > /dev/null; then
echo "Starting syslogd ..." if ! service mariadb status; then
/etc/init.d/rsyslog start echo "ERROR: MySQL isn't running."
grep mysqld /var/log/syslog
exit 1
fi fi
# Give debian-sys-maint access, to avoid complaints later
mysql mysql <<< "GRANT ALL PRIVILEGES on *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '$(awk '/^password/ {print $3; exit}' /etc/mysql/debian.cnf )' WITH GRANT OPTION; FLUSH PRIVILEGES;"
echo "Checking if the IETF database exists at $MYSQLDIR ..."
if [ ! -d $MYSQLDIR/ietf_utf8 ]; then if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
if [ -z "$DATADIR" ]; then echo "WARNING: IETF database seems to be missing; populating it from dump."
echo "DATADIR is not set, but the IETF database needs to be set up -- can't continue, exiting the docker init script." mysqladmin -u root --default-character-set=utf8 create ietf_utf8
exit 1 pushd /mariadb-sys-master || exit
fi mysql -u root < sys_10.sql
ls -l $MYSQLDIR popd || exit
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON *.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
if ! /etc/init.d/mysql status; then /root/src/docker/updatedb
echo "Didn't find the IETF database, but can't set it up either, as MySQL isn't running."
else
echo "Creating database ..."
mysqladmin -u root --default-character-set=utf8 create ietf_utf8
echo "Setting up permissions ..."
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
echo "Fetching database ..."
DUMPDIR=/home/$USER/$DATADIR
wget -N -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz
echo "Loading database ..."
gunzip < $DUMPDIR/ietf_utf8.sql.gz \
| pv --progress --bytes --rate --eta --cursor --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| mysql --user=django --password=RkTkDPFnKpko -s -f ietf_utf8 \
&& rm /tmp/ietf_utf8.sql.gz
fi
fi fi
if ! grep -q ":$GID:$" /etc/group ; then
echo "Creating group entry for GID '$GID' ..."
groupadd -g "$GID" "$USER"
fi
if ! id -u "$USER" &> /dev/null; then
echo "Creating user '$USER' ..."
useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER
echo "$USER:$USER" | chpasswd
fi
VIRTDIR="/opt/home/$USER/$TAG"
echo "Checking that there's a virtual environment for $TAG ..."
if [ ! -f $VIRTDIR/bin/activate ]; then
echo "Setting up python virtualenv at $VIRTDIR ..."
mkdir -p $VIRTDIR
python3.6 -m venv $VIRTDIR
echo -e "
# This is from $VIRTDIR/bin/activate, to activate the
# datatracker virtual python environment on docker container entry:
" >> /etc/bash.bashrc
cat $VIRTDIR/bin/activate >> /etc/bash.bashrc
cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc
else
echo "Using virtual environment at $VIRTDIR"
fi
echo "Activating the virtual python environment ..."
. $VIRTDIR/bin/activate
if ! $VIRTDIR/bin/python -c "import django"; then
echo "Installing requirements ..."
pip install --upgrade pip
reqs=/home/$USER/$CWD/requirements.txt
if [ ! -f $reqs ]; then
echo " Using $reqs"
pip install -r $reqs
else
echo " Didn't find $reqs"
echo " Using /usr/local/share/datatracker/requirements.txt"
pip install -r /usr/local/share/datatracker/requirements.txt
fi
fi
if [ ! -f /home/$USER/$CWD/ietf/settings_local.py ]; then
echo "Setting up a default settings_local.py ..."
cp /home/$USER/$CWD/docker/settings_local.py /home/$USER/$CWD/ietf/settings_local.py
fi
for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do
dir="/home/$USER/$CWD/$sub"
if [ ! -d "$dir" ]; then
echo "Creating dir $dir"
mkdir -p "$dir";
fi
done
for sub in \ for sub in \
nomcom_keys/public_keys \ test/id \
developers/ietf-ftp \ test/staging \
developers/ietf-ftp/bofreq \ test/archive \
developers/ietf-ftp/charter \ test/rfc \
developers/ietf-ftp/conflict-reviews \ test/media \
developers/ietf-ftp/internet-drafts \ test/wiki/ietf \
developers/ietf-ftp/rfc \ data/nomcom_keys/public_keys \
developers/ietf-ftp/status-changes \ data/developers/ietf-ftp \
developers/ietf-ftp/yang/catalogmod \ data/developers/ietf-ftp/bofreq \
developers/ietf-ftp/yang/draftmod \ data/developers/ietf-ftp/charter \
developers/ietf-ftp/yang/ianamod \ data/developers/ietf-ftp/conflict-reviews \
developers/ietf-ftp/yang/invalmod \ data/developers/ietf-ftp/internet-drafts \
developers/ietf-ftp/yang/rfcmod \ data/developers/ietf-ftp/rfc \
developers/www6s \ data/developers/ietf-ftp/status-changes \
developers/www6s/staging \ data/developers/ietf-ftp/yang/catalogmod \
developers/www6s/wg-descriptions \ data/developers/ietf-ftp/yang/draftmod \
developers/www6s/proceedings \ data/developers/ietf-ftp/yang/ianamod \
developers/www6/ \ data/developers/ietf-ftp/yang/invalmod \
developers/www6/iesg \ data/developers/ietf-ftp/yang/rfcmod \
developers/www6/iesg/evaluation \ data/developers/www6s \
data/developers/www6s/staging \
data/developers/www6s/wg-descriptions \
data/developers/www6s/proceedings \
data/developers/www6/ \
data/developers/www6/iesg \
data/developers/www6/iesg/evaluation \
; do ; do
dir="/home/$USER/$CWD/data/$sub" dir="/root/src/$sub"
if [ ! -d "$dir" ]; then if [ ! -d "$dir" ]; then
echo "Creating dir $dir" echo "Creating dir $dir"
mkdir -p "$dir"; mkdir -p "$dir";
chown "$USER" "$dir"
fi fi
done done
if [ ! -f "/home/$USER/$CWD/test/data/draft-aliases" ]; then if [ ! -f /root/src/ietf/settings_local.py ]; then
echo "Generating draft aliases ..." echo "Setting up a default settings_local.py ..."
ietf/bin/generate-draft-aliases } cp /root/src/docker/settings_local.py /root/src/ietf/settings_local.py
fi fi
if [ ! -f "/home/$USER/$CWD/test/data/group-aliases" ]; then python -m smtpd -n -c DebuggingServer localhost:2025 &
echo "Generating group aliases ..." echo
ietf/bin/generate-wg-aliases }
fi
chown -R $USER /opt/home/$USER if [ -z "$*" ]; then
chmod -R g+w /usr/local/lib/ # so we can patch libs if needed bash
cd "/home/$USER/$CWD" || cd "/home/$USER/"
echo "Done!"
if ! echo "$LANG" | grep "UTF-8"; then
echo ""
echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc"
else else
echo "LANG=$LANG" bash -c "$*"
fi fi
HOME=/opt/home/$USER service mariadb stop
service rsyslog stop
su -p $USER

14
docker/install-extras
View file

@ -1,14 +0,0 @@
#!/bin/bash
export DEBIAN_FRONTEND=noninteractive
sudo su - -c "apt-get update \
&& apt-get install -qy \
apt-file \
apt-show-versions \
graphviz \
ghostscript \
apache2-utils \
chromium-driver \
enscript \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*"

13
docker/mailserver
View file

@ -1,13 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
$progdir/run python -m smtpd -n -c DebuggingServer localhost:2025

205
docker/run
View file

@ -1,13 +1,13 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.20
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo $parent | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
function usage() { function usage() {
@ -30,10 +30,10 @@ DESCRIPTION
EOF EOF
echo -e "OPTIONS" echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ] || [ "$(uname)" = "Darwin" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi fi
cat <<EOF cat <<EOF
@ -47,7 +47,7 @@ COPYRIGHT
Copyright (c) 2015 IETF Trust and the persons identified as authors of Copyright (c) 2015 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. License 'Simplified BSD', as specified
in http://opensource.org/licenses/BSD-3-Clause. in http://opensource.org/licenses/BSD-3-Clause.
EOF EOF
} }
@ -58,10 +58,6 @@ function die() {
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
@ -75,48 +71,32 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
# Option parsing # Option parsing
# Options # Options
shortopts=dhi:m:Mp:r:t:vVu:cC shortopts=hp:vVcC
longopts=download-data,help,ietfdb-url=,mysqldata=,no-mysqldir,port=,docker-repo=,tag=,verbose,version,user=,cached,no-cached longopts=help,port=,version,cached,no-cached
# Default values # Default values
MYSQLDIR=$parent/data/mysql MYSQLDIR=$parent/data/mysql
FILEDIR=$parent/../data/
PORT=8000 PORT=8000
REPO="ietf/datatracker-environment" REPO="ietf/datatracker-environment"
DBURL=https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2 CACHED=':cached'
WHO=$(whoami)
CACHED=''
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args" eval set -- "$args"
sed="sed -r"
else else
# Darwin, BSDs # Darwin, BSDs
args=$(getopt -o$shortopts $SV $*) args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args set -- $args
sed="sed -E"
if [ "$(uname)" = "Darwin" ]; then
CACHED=':cached'
fi
fi fi
while true ; do while true ; do
case "$1" in case "$1" in
-c| --cached) CACHED=':cached';; # Use cached disk access to reduce system load -c| --cached) CACHED=':cached';; # Use cached disk access to reduce system load
-C| --no-cached) CACHED='';; # Use fully synchronized disk access -C| --no-cached) CACHED=':consistent';; # Use fully synchronized disk access
-d| --download-data) DOWNLOAD=1;; # Download and set up the database files
-h| --help) usage; exit;; # Show this help, then exit -h| --help) usage; exit;; # Show this help, then exit
-f| --filedir) FILEDIR=$2; shift;; # Set the desired location of drafts, charters etc.
-i| --ietfdb-url) DBURL=$2; shift;; # Use an alternative database tarball URL
-m| --mysqldir) MYSQLDIR=$2; shift;; # Set the desired location of MySQL's database files
-p| --port) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT -p| --port) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
-r| --docker-repo) REPO=$2; shift;; # Use the given docker repository, instead of the default
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the svn branch name
-u| --user) WHO=$2; shift;; # Run the container as the specified user
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit -V| --version) version; exit;; # Show program version, then exit
--) shift; break;; --) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";; *) die "Internal error, inconsistent option specification: '$1'";;
@ -125,163 +105,14 @@ while true ; do
done done
if [ -z "$TAG" ]; then if [ -z "$TAG" ]; then
TAG=$(basename $(svn info $parent | grep ^URL | awk '{print $2}')) TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}')")
fi fi
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# The program itself
[ -f /proc/1/cgroups ] && grep 'docker' /proc/1/cgroups && die "It looks like you're inside docker already ..." echo "Starting a docker container for '$REPO:$TAG'."
mkdir -p "$MYSQLDIR"
if [ "$(uname)" = "Darwin" ]; then docker run -ti -p "$PORT":8000 -p 33306:3306 \
APP="/Applications/Docker.app" -v "$parent:/root/src$CACHED" \
CMD="open -a" -v "$MYSQLDIR:/var/lib/mysql:delegated" \
elif [ "$(uname)" = "Linux" ]; then "$REPO:$TAG" "$@"
echo "Running on Linux."
elif [[ $(uname) =~ CYGWIN.* ]]; then
echo "Running under Cygwin."
CMD="echo"
MYSQLDIR=$(echo $MYSQLDIR | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
WHO=$(echo $WHO | sed -e 's/^.*\\//' | tr -d \\r)
DRIVE=$(echo $USERPROFILE | sed -e 's/\(.\).*/\1/' -e 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefdhijklmnopqartuvwxyz/')
HOME=$DRIVE$(echo $USERPROFILE | sed -e 's/\\/\//g' -e's/.\(.*\)/\1/')
echo "Using home dir $HOME"
PWD=$(echo $PWD | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
if [ "${PWD#$HOME}" = "$PWD" ]; then
die "You must work inside your home directory ($HOME)"
fi
CGWPARENT=$(echo $parent | sed -e 's/^\(.\)\:/\/cygdrive\/\1/')
ICSFILES=$(/usr/bin/find $CGWPARENT/vzic/zoneinfo/ -name '*.ics' -print)
for ICSFILE in $ICSFILES
do
LINK=$(head -n1 $ICSFILE | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
if [ "$LINK" ]; then
WDIR=$(dirname $ICSFILE)
echo "Replacing $(basename $ICSFILE) with $LINK"
cp -f $WDIR/$LINK $ICSFILE
fi
done
else
die "This script does not have support for your architecture ($(uname)); sorry :-("
fi
WHOUID=$(id -u $WHO)
WHOGID=$(id -g $WHO)
if [ "$(uname)" = "Linux" ]; then
echo "Not trying to start a virtual docker machine on Linux"
elif [[ $(uname) =~ CYGWIN.* ]]; then
if ! docker info 1> /dev/null 2>&1; then
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
die "Don't know how to start docker when running under Cygwin"
fi
TAG=$(echo $TAG | tr -d \\r)
URL="http://localhost:$PORT/"
elif [ -e "$APP" ]; then
info=$(docker info 2>&1 || true)
if ! docker info 1> /dev/null 2>&1; then
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
echo "Will attempt to start docker by doing '\$ $CMD $APP':"
$CMD $APP
declare -i count
printf "Waiting for docker engine .."
while true; do
printf "."
sleep 2
if docker info >/dev/null 2>&1; then break; fi
count=$(( $count + 1))
if [ $count -gt 10 ]; then
die "No contact with docker engine; bailing out."
fi
done
fi
URL="http://localhost:$PORT/"
else
if [ $($machine status default | tr "A-Z" "a-z" ) != "running" ]; then
echo "The docker VM doesn't seem to be running; will attempt to start it by doing '\$ $machine start':"
$machine start || die "Failed taking up the Docker VM"
fi
if [ -f "$machine" ]; then
if [ $($machine status default | tr "A-Z" "a-z") = "running" ]; then
tmpfile=$(mktemp docker.run.XXXXXXXX)
if $machine env 2>/dev/null | grep DOCKER_ > $tmpfile; then
mv $tmpfile ~/.docker-info
elif printenv | grep DOCKER_ > $tmpfile; then
mv $tmpfile ~/.docker-info
else
rm $tmpfile
die "Failed setting the appropriate DOCKER_* environment variables."
fi
. ~/.docker-info
else
rm -f ~/.docker-info
fi
ip=$($machine ip)
URL="http://$ip:$PORT/"
fi
fi
image=$(docker ps | grep "$REPO:$TAG" | awk '{ print $1 }')
if [ "$image" ]; then
if [ "$*" ]; then
echo "Running 'cd ~/${parent#$HOME/}; $*'"
docker exec -u "$WHO" -t $image bash -i -c "cd; cd \"${parent#$HOME/}\"; $*"
else
docker exec -u "$WHO" -ti $image bash -i
fi
else
echo ""
echo "Starting a docker container for '$TAG'."
if [ -n "$DOWNLOAD" ]; then
(
cd "$(dirname $MYSQLDIR)"
wget -N "$DBURL" && tar xjf ietf_utf8.bin.tar.bz2 && chmod -R go+rwX mysql
)
[ -d "$MYSQLDIR" ] || die "The download seems to have failed; still no $MYSQLDIR. Giving up."
else
[ -d "$MYSQLDIR" ] || die "Expected $MYSQLDIR to exist, but it\ndidn't. Use '$program -d' to download and unpack the database."
fi
if ! docker images "$REPO" | grep -q "$TAG"; then
echo "Fetching docker image '$REPO:$TAG'"
if ! docker pull "$REPO:$TAG"; then
docker pull "$REPO:latest" || die "Failed to pull down the '$REPO:latest' docker image"
id=$(docker images "$REPO" | grep latest | awk '{print $3}')
echo "Tagging $REPO:latest as $REPO:$TAG for use as environment for this branch."
docker tag $id "$REPO:$TAG"
fi
fi
echo -e "\nThe web interface for 'runserver' should appear on $URL\n"
echo -e "User $WHO ($WHOUID:$WHOGID)"
if [ -z "$MYSQLDIR" ]; then
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
-e USER="$WHO" -e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" \
-e TAG="$TAG" -e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
"$REPO:$TAG" "$@"
else
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
-v "$MYSQLDIR:/var/lib/mysql" -e USER="$WHO" \
-e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" -e TAG="$TAG" \
-e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
"$REPO:$TAG" "$@"
fi
echo ""
echo "Committing changes in the container to an image:"
image=$( docker images -q $REPO:$TAG)
latest=$(docker ps -lq -f "ancestor=$image")
docker commit $latest $REPO:$TAG
echo ""
echo "Cleaning up containers and images"
docker rm $latest
DANGLING=$( docker images -f dangling=true -q )
if [ -n "$DANGLING" ]; then
echo "Dangling images: $DANGLING"
docker rmi -f $DANGLING
fi
fi

26
docker/setprompt
View file

@ -1,26 +0,0 @@
# set a fancy prompt
PS_TIME="\A "
if [ "$(whoami)" = "root" ]; then
PS_COLOUR='\[\033[1;31m\]'
PS_END="\[\033[m\]"
else
PS_COLOUR="\[\033[1;34m\]"
PS_END="\[\033[m\]"
fi
if [ "$TERM" = xterm ]; then
PS_XTERM="\[\033]0;\]\h:\w\007"
else
PS_XTERM=""
fi
COLOUR_RED='\[\033[1;31m\]'
COLOUR_BLK='\[\033[1;30m\]'
if [ "$SHLVL" -gt 1 ]; then
PS_SHLVL="$(eval "printf '>%.0s' {2..$SHLVL}") "
else
PS_SHLVL=""
fi
PS1="$PS_XTERM\n$COLOUR_RED$PS_SHLVL$PS_COLOUR$PS_TIME$COLOUR_RED$PS_COLOUR${VIRTUAL_ENV:+$COLOUR_RED($(basename $VIRTUAL_ENV))$PS_COLOUR }\w\n\u @ $COLOUR_BLK\h$PS_COLOUR \\$ $PS_END"

114
docker/setupdb
View file

@ -1,114 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
function usage() {
cat <<EOF
NAME
$program - Set up a local copy of the IETF database MySQL files
SYNOPSIS
$program [OPTIONS] ARGS
DESCRIPTION
This script downloads a prebuilt copy of the IETF database MySQL
files, ready for mapping into the /var/lib/mysql/ directory of the
ietf/database-environment Docker image.
EOF
echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF
FILES
AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com>
COPYRIGHT
Copyright (c) 2015 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified
in http://opensource.org/licenses/BSD-3-Clause.
EOF
}
# ----------------------------------------------------------------------
function die() {
echo -e "\n$program: error: $*" >&2
exit 1
}
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() {
echo -e "$program $version"
}
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=hvV
longopts=help,verbose,version
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do
case "$1" in
-h| --help) usage; exit;; # Show this help, then exit
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit
--) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";;
esac
shift
done
# ----------------------------------------------------------------------
# The program itself
[ -n "$MYSQLDIR" ] || MYSQLDIR=$parent/data/mysql
[ -n "$URL" ] || URL=rsync.ietf.org::dev.db/ietf_utf8.bin.tar.bz2
cd $(dirname $MYSQLDIR)
echo "Downloading a MySQL database image ..."
set $(rsync --version | head -n1 | awk '{print $3}' | tr '.' ' ')
maj=$1; min=$2; fix=$3
if [ "$maj" -ge "3" -a "$min" -ge "1" ]; then
rsync --info=progress2 $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
else
rsync -v $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
fi

72
docker/updatedb
View file

@ -1,12 +1,12 @@
#!/bin/bash #!/bin/bash
# -*- indent-with-tabs: 1 -*- # -*- indent-with-tabs: 1 -*-
version=0.10 version=0.20
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
@ -26,9 +26,9 @@ DESCRIPTION
EOF EOF
echo -e "OPTIONS" echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /' grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi fi
cat <<EOF cat <<EOF
@ -42,21 +42,17 @@ COPYRIGHT
Copyright (c) 2015 IETF Trust and the persons identified as authors of Copyright (c) 2015 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. License 'Simplified BSD', as specified
in http://opensource.org/licenses/BSD-3-Clause. in http://opensource.org/licenses/BSD-3-Clause.
EOF EOF
} }
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
@ -71,24 +67,21 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
# Options # Options
shortopts=DLZhqvV shortopts=DLZhqvV
longopts=no-download,no-load,no-zap,help,quiet,verbose,version longopts=no-download,no-load,no-zap,help,version
LOAD=1 LOAD=1
DOWNLOAD=1 DOWNLOAD=1
DROP=1 DROP=1
QUIET=""
if [ "$(uname)" = "Linux" ]; then if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n "$program" -- $SV "$@") args=$(getopt -o "$shortopts" --long "$longopts" -n "$program" -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args" eval set -- "$args"
sed="sed -r"
else else
# Darwin, BSDs # Darwin, BSDs
args=$(getopt -o$shortopts $SV $*) args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args set -- $args
sed="sed -E"
fi fi
while true ; do while true ; do
@ -97,8 +90,6 @@ while true ; do
-L| --no-load) LOAD=""; ;; # Don't load the database -L| --no-load) LOAD=""; ;; # Don't load the database
-Z| --no-zap) DROP="";; # Don't drop new tables -Z| --no-zap) DROP="";; # Don't drop new tables
-h| --help) usage; exit;; # Show this help, then exit -h| --help) usage; exit;; # Show this help, then exit
-q| --quiet) QUIET="-q";; # Don't show progress
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit -V| --version) version; exit;; # Show program version, then exit
--) shift; break;; --) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";; *) die "Internal error, inconsistent option specification: '$1'";;
@ -109,50 +100,27 @@ done
# ---------------------------------------------------------------------- # ----------------------------------------------------------------------
# The program itself # The program itself
echo "Gathering info ..."
MYSQLDIR="$(/usr/sbin/mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
MYSQLDIR=${MYSQLDIR%/}
DATADIR=$parent/data DATADIR=$parent/data
# echo "Checking if MySQL base data exists ..." DUMP=ietf_utf8.sql.gz
# if [ ! -d $MYSQLDIR/mysql ]; then
# die "Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
# fi
# echo "Checking if the IETF database exists at $MYSQLDIR ..."
# if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
# echo "Creating database ..."
# mysqladmin -u root --default-character-set=utf8 create ietf_utf8
#
# echo "Setting up permissions ..."
# mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
# fi
if [ "$DOWNLOAD" ]; then if [ "$DOWNLOAD" ]; then
echo "Fetching database dump ..." echo "Fetching database dump..."
wget $QUIET -N -P $DATADIR https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz || die "No new data, quitting." rsync --info=progress2 rsync.ietf.org::dev.db/$DUMP "$DATADIR"
fi fi
if [ "$LOAD" ]; then if [ "$LOAD" ]; then
echo "Loading database ..." echo "Loading database..."
if [ -z "$QUIET" ]; then SIZE=$(pigz --list "$DATADIR/$DUMP" | tail -n 1 | awk '{ print $2 }')
gunzip < $DATADIR/ietf_utf8.sql.gz \ pigz -d < "$DATADIR/$DUMP" \
| pv --progress --bytes --rate --eta --size $(gzip --list --quiet $DATADIR/ietf_utf8.sql.gz | awk '{ print $2 }') \ | pv --progress --bytes --rate --eta --size "$SIZE" \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \ | sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| $parent/ietf/manage.py dbshell | "$parent/ietf/manage.py" dbshell
else
gunzip < $DATADIR/ietf_utf8.sql.gz \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| $parent/ietf/manage.py dbshell
fi
fi fi
if [ "$DROP" ]; then if [ "$DROP" ]; then
echo "Dropping tables not in the dump (so migrations can succeed) ..." echo "Dropping tables not in the dump (so migrations can succeed)..."
diff <(zcat $DATADIR/ietf_utf8.sql.gz | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \ diff <(pigz -d -c "$DATADIR/$DUMP" | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
<($parent/ietf/manage.py dbshell <<< 'show tables;' | tail -n +2) \ <("$parent/ietf/manage.py" dbshell <<< 'show tables;' | tail -n +2) \
| grep '^>' | awk '{print "drop table if exists", $2, ";";}' \ | grep '^>' | awk '{print "drop table if exists", $2, ";";}' \
| tee >(cat >&2) | $parent/ietf/manage.py dbshell | tee >(cat >&2) | "$parent/ietf/manage.py" dbshell
fi fi