altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Merged branch ^/personal/lars/7.39.1.dev0@19465 from lars@eggert.org:

Browse source 
Rework the docker container.
 - Legacy-Id: 19507
This commit is contained in:
Robert Sparks 2021-11-01 16:30:43 +00:00
parent 59597794b7 2b80cf7e1d
commit 2b478da42e
14 changed files with 428 additions and 1124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

269
docker/Dockerfile
View file

@ -1,185 +1,132 @@
# -*- shell-mode -*- # This is a Dockerfile with everything in it to run the IETF datatracker.
# This file is a docker (https://www.docker.com/what-docker) recipe, which can be used to build
# a docker image which is ready to run a datatracker in development mode.
# #
# It is used to build an image (once you've installed docker) using a command like this (assuming # If you make changes to the datatracker that add new dependencies (python
# suitable replacement of $variables: # packages or otherwise), you need to rebuild this image to make them
# available. Do this in the top-level directory of your datatracker source
# tree:
# #
# $ docker build -t $yourdockerid/datatracker:$version # docker/build
# #
# To use a pre-built image, assuming we're on OS X and have a checked-out datatracker repository # You can then execute the datatracker like this (also from the top-level
# at /Users/$login/src/6.8.1.dev0, you would start (again assuming you've installed docker) # datatracker source directory):
# a container from an image, as follows: #
# # docker/run
# $ docker run -ti --name=$containername -v /Users/$login:/home/$login levkowetz/datatracker:$version /bin/bash
#
# This maps your home directory to /home/$login in the container, and starts it running /bin/bash.
#
# In this first version, the docker environment is set up so that tests will run successfully,
# but the database has *not* been loaded with a dump, and supporting files (drafts, charters, etc.)
# have *not* been downloaded.
FROM dyne/devuan:beowulf FROM ubuntu:hirsute
LABEL maintainer="Henrik Levkowetz <henrik@levkowetz.com>" LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
# Default django runserver port # Default django runserver port
EXPOSE 8000 EXPOSE 8000
# Default mysqld/mariadb port
EXPOSE 3306
# Run apt-get noninteractive
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
ENV DEVUAN_FRONTEND=noninteractive RUN apt-get -y update && \
# apt-get upgrade is normally not a good idea, but this is a dev container
# Uncomment this to be able to install and run apt-show-versions: apt-get upgrade && \
RUN rm -v /etc/apt/apt.conf.d/docker-compress # Install all dependencies that are available as packages
RUN rm -v /var/lib/apt/lists/*lz4 apt-get -y install --no-install-recommends \
apache2-utils \
RUN apt-get update --allow-releaseinfo-change apt-file \
RUN apt-get install -qy apt-transport-https apt-utils \
curl \
# Use backports, updates, and security updates enscript \
RUN echo "deb http://deb.devuan.org/merged beowulf main contrib non-free" > /etc/apt/sources.list gcc \
RUN echo "deb http://deb.devuan.org/merged beowulf-security main contrib non-free" >> /etc/apt/sources.list ghostscript \
RUN echo "deb http://deb.devuan.org/merged beowulf-updates main contrib non-free" >> /etc/apt/sources.list git \
RUN echo "deb http://deb.devuan.org/merged beowulf-backports main contrib non-free" >> /etc/apt/sources.list gnupg \
graphviz \
# Remove some excludes for the docker image libmagic-dev \
RUN sed -i -e '/^path-exclude=.*\/groff/d' \ libmariadb-dev \
-e '/^path-exclude=.*\/locale/d' \ locales \
-e '/^path-exclude=.*\/man/d' /etc/dpkg/dpkg.cfg.d/docker-excludes mariadb-server \
npm \
# Install needed packages pigz \
# pv \
# We're not including graphviz and ghostscript, needed for the 3 document python-is-python3 \
# dependency graph tests; they would increase the size of the image by about python3-dev \
# 15%, about 100MB. python3-pip \
# Fetch apt package information, and upgrade to latest package versions
RUN apt-get update
RUN apt-get -qy upgrade
# Install the packages we need
RUN apt-get install -qy \
build-essential \
bzip2 \
ca-certificates \
colordiff \
gawk \
gcc \
ipython \
jq \
less \
libbz2-dev \
libdb5.3-dev \
libexpat1-dev \
libffi-dev \
libgdbm-dev \
libjpeg62-turbo-dev \
liblzma-dev \
libmagic1 \
libmariadbclient-dev \
libncurses5-dev \
libncursesw5-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libsvn1 \
libxml2-dev \
libxslt-dev \
libz-dev \
libffi-dev \
locales \
make \
man \
mariadb-client \
mariadb-server \
openssh-client \
patch \
procps \
pv \
rsync \
rsyslog \ rsyslog \
subversion \ unzip \
sudo \ yang-tools && \
uuid-dev \ # Since snap doesn't work in Docker containers, install chromedriver per
vim \ # https://gist.github.com/varyonic/dea40abcf3dd891d204ef235c6e8dd79#gistcomment-3160722
wget \ curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
xz-utils\ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
zile \ apt-get update -y && \
zlib1g-dev apt-get install -y --no-install-recommends google-chrome-stable && \
CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
DRIVERVER=$(curl "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") && \
curl -L -O -C - "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" && \
unzip chromedriver_linux64.zip -d /bin && \
rm chromedriver_linux64.zip && \
# Install some other packages that are not dependencies but make life easier
apt-get -y install --no-install-recommends \
fish \
less \
nano \
ripgrep \
zsh && \
# Reduce image footprint (not that it matters given the size of the above)
apt-get -y clean && \
rm -rf /var/lib/apt/lists/*
# Postgresql packages # Set locale to en_US.UTF-8
RUN apt-get install -qy \ RUN dpkg-reconfigure locales && \
postgresql-11 \ locale-gen en_US.UTF-8 && \
postgresql-server-dev-11 update-locale LC_ALL en_US.UTF-8
ENV LC_ALL en_US.UTF-8
# Get the key used to sign the libyang repo # Install bower
RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key RUN npm install -g bower
RUN apt-key add - < Release.key
RUN rm Release.key
# Add apt source entry for libyang # Install idnits
RUN echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list
# Update the package defs, and install the desired mysql from the mysql repo
RUN apt-get update
RUN apt-get install -qy libyang1
# This is expected to exist by the mysql startup scripts:
#RUN touch /etc/mysql/debian.cnf
# ------------------------------------------------------------------------------
# Get rid of installation files we don't need in the image, to reduce size
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
# Enable some common locales
RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen
RUN locale-gen
# Remove an rsyslog module that we don't need, which also requires extra permissions
RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf
# Set up root password
RUN echo "root:root" | chpasswd
# MySQL
VOLUME /var/lib/mysql
# idnits and dependencies
ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/ ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/
RUN chmod +rx /usr/local/bin/idnits RUN chmod +rx /usr/local/bin/idnits
# Directory for Mac certs # Install current datatracker python dependencies
RUN mkdir /etc/certificates COPY requirements.txt /
RUN pip install -r /requirements.txt
# # Python 3 # Turn off rsyslog kernel logging (doesn't work in Docker)
# # Comment in if OS does not provide python3.6, which is the current RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
# # production version
ENV PYVER=3.6.10
ENV PYREV=3.6
WORKDIR /usr/src # Allow access to mariadb over the network
RUN wget -q https://www.python.org/ftp/python/$PYVER/Python-$PYVER.tar.xz RUN sed -i 's/127.0.0.1/0.0.0.0/' /etc/mysql/mariadb.conf.d/50-server.cnf
RUN tar xJf Python-$PYVER.tar.xz
RUN rm Python-$PYVER.tar.xz
WORKDIR /usr/src/Python-$PYVER/
RUN ./configure
RUN make
RUN make altinstall
WORKDIR /usr/src
RUN rm -rf /usr/src/Python-$PYVER/
ENV HOSTNAME="datatracker" # Turn on mariadb performance_schema
RUN sed -i 's/\[mysqld\]/\[mysqld\]\nperformance_schema=ON/' /etc/mysql/mariadb.conf.d/50-server.cnf
ENV DDIR="/usr/local/share/datatracker" # Make the mariadb sys schema available for possible installation
RUN mkdir -p $DDIR # We would normally use the next line, but that has a bug:
WORKDIR $DDIR # ADD https://github.com/FromDual/mariadb-sys/archive/master.zip /
# This is the repo that has the PR:
ADD https://github.com/grooverdan/mariadb-sys/archive/refs/heads/master.zip /
RUN unzip /master.zip
RUN rm /master.zip
COPY requirements.txt ./ # Colorize the bash shell
COPY setprompt ./ RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc
# Make a database dump available as part of the image, for if a user doesn't
# have one installed locally yet - this saves a bunch of time then
ADD https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz /
RUN pigz -v -d /ietf_utf8.sql.gz && \
sed -i -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' /ietf_utf8.sql
RUN service mariadb start && \
echo "This sequence will take a long time, please be patient" && \
mysqladmin -u root --default-character-set=utf8 create ietf_utf8 && \
bash -c "cd /mariadb-sys-master && mysql --user root < sys_10.sql" && \
bash -c "mysql --user root ietf_utf8 <<< \"GRANT ALL PRIVILEGES ON *.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;\"" && \
bash -c "mysql --user=django --password=RkTkDPFnKpko -f ietf_utf8 < /ietf_utf8.sql" && \
service mariadb stop && \
rm -rf /ietf_utf8.sql /mariadb-sys-master && \
mv /var/lib/mysql /
# Copy the startup file
COPY docker-init.sh /docker-init.sh COPY docker-init.sh /docker-init.sh
RUN chmod +x /docker-init.sh RUN chmod +x /docker-init.sh
ENTRYPOINT ["/docker-init.sh"]
CMD /bin/bash WORKDIR /root/src
ENTRYPOINT ["/docker-init.sh"]

56
docker/README.md Normal file
View file

@ -0,0 +1,56 @@
# Datatracker Development in Docker
## Getting started
1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred
platform.
2. If you have a copy of the datatracker code checked out already, simply `cd`
to the top-level directory.
If not, check out a datatracker branch as usual. We'll check out `trunk`
below, but you can use any branch:
svn co https://svn.ietf.org/svn/tools/ietfdb/trunk
cd trunk
3. **TEMPORARY:** Replace the contents of the `docker` directory with [Lars'
files](https://svn.ietf.org/svn/tools/ietfdb/personal/lars/7.39.1.dev0/docker/).
4. **TEMPORARY:** Until [Lars'
changes](https://svn.ietf.org/svn/tools/ietfdb/personal/lars/7.39.1.dev0/docker/)
have been merged and a docker image is available for download, you will need
to build it locally:
docker/build
This will take a while, but only needs to be done once.
5. Use the `docker/run` script to start the datatracker container. You will be
dropped into a shell from which you can start the datatracker and execute
related commands as usual, for example
ietf/manage.py runserver 0.0.0.0:8000
to start the datatracker.
You can also pass additional arguments to `docker/run`, in which case they
will be executed in the container (instead of a shell being started.)
If you do not already have a copy of the IETF database available in the
`data` directory, one will be downloaded and imported the first time you run
`docker/run`. This will take some time.
Once the datatracker has started, you should be able to open
[http://localhost:8000](http://localhost:8000) in a browser and see the
landing page.
## Troubleshooting
- If the database fails to start, the cause is usually an incompatibility
between the database that last touched the files in `data/mysql` and the
database running inside the docker container.
The solution is to blow away your existing database (`rm -rf data/mysql`). A
fresh copy will be retrieved and imported next time you do `docker/run`, which
should resolve this issue.

98
docker/README.rst
View file

@ -1,98 +0,0 @@
==============================================================================
Datatracker Development in a Docker Container (beta)
==============================================================================
Intro
=====
Docker_ is a toolkit which lets you package software together with its
dependencies in lightweight containers, and run it in isolated virtual
environments.
During and just after IETF-94 I've spent quite a bit of time setting up a
docker image which provides the dependencies needed to run the datatracker,
and it's now available for beta testing. Hopefully this should make it
substantially easier to get started with datatracker development.
Steps
=====
1. Set up Docker on your preferred platform. Official installers exist for
many Linux flavours, OS X, Windows and Cloud services. Here's the full `List
of Installation Instructions`_.
Docker containers require the services of an underlying Linux API, which
means that on OS X and Windows, these have to be provided by a virtual
machine which runs a minimal Linux image. The virtual machine used on
non-Linux platforms is commonly VirtualBox. On Linux kernels with version
3.8 or later, no virtual machine is needed, as the docker images can be
fully supported with the native kernel services.
Please follow the Docker installations all the way through to successfully
running the ``hello-world`` example in a terminal window ( ``$ docker run
hello-world``).
2. Check out your datatracker branch as usual, in a suitable directory.
We'll assume ``~/src/dt/`` here, and assume you are ``'coder'``::
~/src/dt/ $ svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/coder/6.8.2.dev0
3. In the checked-out working copy, you'll find a ``docker/`` directory and a
``data/`` directory at the top level. We're first going to set up a copy of
the MySQL database files under the ``data/`` directory.
There is a command in the ``docker/`` directory, ``setupdb`` which will do
this for you, or you can do it manually.
Either run::
~/src/dt/6.8.2.dev0/ $ docker/setupdb
or do this step-by-step: fetch down a pre-built copy of the datatracker
database, place it in the ``data`` directory, unpack it, and fix
permissions::
~/src/dt/6.8.2.dev0/ $ cd data
~/src/dt/6.8.2.dev0/data/ $ wget https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
~/src/dt/6.8.2.dev0/data/ $ tar xjf ietf_utf8.bin.tar.bz2
~/src/dt/6.8.2.dev0/data/ $ chmod -R go+rwX mysql
4. In the ``docker/`` directory you'll also find a wrapper script named
``'run'``. We will be using the wrapper to run a pre-built docker image
fetched from the docker hub::
~/src/dt/6.8.2.dev0/ $ docker/run
This will pull down the latest docker ietf/datatracker-environment image,
start it up with appropriate settings, map the internal ``/var/lib/mysql/``
directory to the external ``data/mysql/`` directory where we placed the
database, set up a python virtualenv for you, install some dependencies,
and drop you in a bash shell where you can run the datatracker.
6. You are now ready to run the tests::
(virtual) $ ietf/manage.py test --settings=settings_sqlitetest
and then start the dev server::
(virtual) $ ietf/manage.py runserver 0.0.0.0:8000
Note the IP address ``0.0.0.0`` used to make the dev server bind to all
addresses. The internal port 8000 has been mapped to port 8000 externally,
too. In order to find the IP address of the VirtualBox, run ``'$
docker-machine ip'`` *outside* the virtual environment::
~/src/dt/6.8.2.dev0/ $ docker-machine ip
192.168.59.103
~/src/dt/6.8.2.dev0/ $ open http://192.168.59.103:8000/
.. _Docker: https://www.docker.com/
.. _`List of Installation Instructions`: https://docs.docker.com/v1.8/installation/
.. _VirtualBox: https://www.virtualbox.org/

110
docker/build
View file

@ -1,126 +1,92 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.20
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
export LANG=C
# ----------------------------------------------------------------------
function usage() { function usage() {
cat <<EOF cat <<EOF
NAME NAME
$program - Build a docker datatracker image. $program - Build a datatracker docker image
SYNOPSIS SYNOPSIS
$program [OPTIONS] ARGS $program [OPTIONS]
DESCRIPTION DESCRIPTION
This script builds a Ubuntu-based docker image that has been
This script builds a debian-based docker image which has been
set up with the dependencies needed to easily run the IETF set up with the dependencies needed to easily run the IETF
datatracker in development mode. It uses docker/Dockerfile; datatracker in development mode. It uses docker/Dockerfile;
i.e., the Dockerfile in the same directory as this script. i.e., the Dockerfile in the same directory as this script.
It assumes that the user has upload rights for the docker
ietf/datatracker-environment repository, in order to push the
image.
OPTIONS
EOF EOF
echo -e "OPTIONS" grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF cat <<EOF
FILES
AUTHOR AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com> Written by:
Henrik Levkowetz, <henrik@levkowetz.com>
Lars Eggert, <lars@eggert.org>
COPYRIGHT COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of Copyright (c) 2016 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. Redistribution and use in source and
in http://opensource.org/licenses/BSD-3-Clause. binary forms, with or without modification, is permitted pursuant to,
and subject to the license terms contained in, the Revised BSD
EOF License set forth in Section 4.c of the IETF Trust’s Legal Provisions
Relating to IETF Documents(https://trustee.ietf.org/license-info).
EOF
} }
# ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
} }
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=hlt:vV
longopts=help,local,tag=,verbose,version
# Default values # Default values
BRANCH=$(svn log -v ^/tags -l 2 | grep 'A /tags/[1-9]' | awk '{print $2}') IMAGE=ietf/datatracker-environment
TAG=${BRANCH##*/} TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}' | tr -d '\r')")
LOCAL="" LOCAL=1
if [ "$(uname)" = "Linux" ]; then # Option parsing
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") shortopts=hut:V
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi args=$(getopt -o$shortopts $*)
eval set -- "$args" if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
sed="sed -r" set -- $args
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do while true ; do
case "$1" in case "$1" in
-h| --help) usage; exit;; # Show this help, then exit -h) usage; exit;; # Show this help, then exit
-l| --local) LOCAL=1;; # Don't upload -u) LOCAL=0;; # Upload image to repository after build
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the latest svn tags name -t) TAG=$2; shift;; # Use this docker image tag
-v| --verbose) VERBOSE=1;; # Be more talkative -V) version; exit;; # Show program version, then exit
-V| --version) version; exit;; # Show program version, then exit --) shift; break;;
--) shift; break;; *) die "Internal error, inconsistent option specification: '$1'";;
*) die "Internal error, inconsistent option specification: '$1'";;
esac esac
shift shift
done done
# ----------------------------------------------------------------------
# The program itself # The program itself
docker rmi -f $IMAGE:trunk 2>/dev/null || true
if [ "$(uname)" == "Linux" ]; then docker build --progress plain -t "$IMAGE:$TAG" docker/
die "Didn't expect to run this script on Linux -- are you inside docker?" docker tag "$(docker images -q $IMAGE | head -n 1)" $IMAGE:latest
fi
docker rmi -f ietf/datatracker-environment:trunk || true
docker build -t ietf/datatracker-environment:$TAG docker/
docker tag $(docker images -q ietf/datatracker-environment | head -n 1) ietf/datatracker-environment:latest
if [ -z "$LOCAL" ]; then if [ -z "$LOCAL" ]; then
docker push ietf/datatracker-environment:latest docker push $IMAGE:latest
docker push ietf/datatracker-environment:$TAG docker push "$IMAGE:$TAG"
fi fi

83
docker/copydb
View file

@ -1,118 +1,89 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.11
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
export LANG=C
# ----------------------------------------------------------------------
function usage() { function usage() {
cat <<EOF cat <<EOF
NAME NAME
$program - Make a tarball of the MySQL database files and upload it. $program - Make a tarball of the MySQL database files and upload it
SYNOPSIS SYNOPSIS
$program [OPTIONS] $program [OPTIONS]
DESCRIPTION DESCRIPTION
This script creates a compressed tarball from the MySQL database files This script creates a compressed tarball from the MySQL database files
on disk, and uploads it to the ietf datatracker developer area on on disk, and uploads it to the ietf datatracker developer area on
www.ietf.org. www.ietf.org.
It is intended to be used with the docker datatracker environment, after It is intended to be used with the docker datatracker environment, after
you have set up the database with docker/setupdb, started the docker you have set up the database with docker/setupdb, started the docker
image with docker/run, and updated the database with docker/updatedb. image with docker/run, and updated the database with docker/updatedb.
To use it, exit from the docker container, to make sure that mysqldb To use it, exit from the docker container, to make sure that mysqldb
isn't running and all the files consistent and available for copy. Then isn't running and all the files consistent and available for copy. Then
run docker/$program outside the docker container. You need to have ssh run docker/$program outside the docker container. You need to have ssh
access to www.ietf.org in order for the scp command to succeed. access to www.ietf.org in order for the scp command to succeed.
OPTIONS
EOF EOF
echo -e "OPTIONS" grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF cat <<EOF
FILES
AUTHOR AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com> Written by:
Henrik Levkowetz, <henrik@levkowetz.com>
Lars Eggert, <lars@eggert.org>
COPYRIGHT COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of Copyright (c) 2016 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. Redistribution and use in source and
in http://opensource.org/licenses/BSD-3-Clause. binary forms, with or without modification, is permitted pursuant to,
and subject to the license terms contained in, the Revised BSD
EOF License set forth in Section 4.c of the IETF Trust’s Legal Provisions
Relating to IETF Documents(https://trustee.ietf.org/license-info).
EOF
} }
# ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
} }
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing # Option parsing
shortopts=hV
# Options args=$(getopt -o$shortopts $*)
shortopts=hvV if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
longopts=help,verbose,version set -- $args
# Default values
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do while true ; do
case "$1" in case "$1" in
-h| --help) usage; exit;; # Show this help, then exit -h) usage; exit;; # Show this help, then exit
-v| --verbose) VERBOSE=1;; # Be more talkative -V) version; exit;; # Show program version, then exit
-V| --version) version; exit;; # Show program version, then exit --) shift; break;;
--) shift; break;; *) die "Internal error, inconsistent option specification: '$1'";;
*) die "Internal error, inconsistent option specification: '$1'";;
esac esac
shift shift
done done
# ----------------------------------------------------------------------
# The program itself # The program itself
if [ -e "/.dockerenv" -o -n "$(grep -s '/docker/' /proc/self/cgroup)" ]; then if [ -e "/.dockerenv" -o -n "$(grep -s '/docker/' /proc/self/cgroup)" ]; then
die "It looks as if you're running inside docker -- please quit docker first." die "It looks as if you're running inside docker -- please quit docker first."
fi fi
@ -123,4 +94,4 @@ cd $workdir
echo "Building tarfile ..." echo "Building tarfile ..."
tar cjf ietf_utf8.bin.tar.bz2 mysql tar cjf ietf_utf8.bin.tar.bz2 mysql
echo "Copying tarfile to ietfa.amsl.com ..." echo "Copying tarfile to ietfa.amsl.com ..."
scp ietf_utf8.bin.tar.bz2 ietfa.amsl.com:/a/www/www6s/lib/dt/sprint/ scp ietf_utf8.bin.tar.bz2 ietfa.amsl.com:/a/www/www6s/lib/dt/sprint/

15
docker/devserver
View file

@ -1,15 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
$progdir/run ietf/manage.py runserver 0.0.0.0:8000

238
docker/docker-init.sh
View file

@ -1,195 +1,91 @@
#!/bin/bash #!/bin/bash
# A little bit of setup MYSQLDIR=/var/lib/mysql
export LANG=en_GB.UTF-8
echo "Gathering info ..." if [ ! -d "$MYSQLDIR" ]; then
MYSQLDIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" echo "WARNING: Expected the directory $MYSQLDIR to exist."
if [ ! "$USER" ]; then exit 1
echo "Environment variable USER is not set -- will set USER='django'."
USER="django"
fi
if [ ! "$UID" ]; then
echo "Environment variable UID is not set -- will set UID='1000'."
UID="1000"
fi
if [ ! "$GID" ]; then
echo "Environment variable GID is not set -- will set GID='1000'."
GID="1000"
fi
if [ ! "$TAG" ]; then
echo "Environment variable TAG is not set -- will set TAG='datatracker'."
TAG="datatracker"
fi
echo "User $USER ($UID:$GID)"
echo "Checking if MySQL base data exists ..."
if [ ! -d $MYSQLDIR/mysql ]; then
echo "WARNING: Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
fi fi
echo "Setting up the 'mysql' user for database file access ..." service rsyslog start
MYSQL_TARGET_GID=$(stat -c "%g" $MYSQLDIR/mysql)
if ! grep -q ":$MYSQL_TARGET_GID:$" /etc/group; then
groupadd -g $MYSQL_TARGET_GID mysqldata
fi
usermod -a -G $MYSQL_TARGET_GID mysql
echo "Checking if MySQL is running ..." if [ -z "$(ls -A $MYSQLDIR/mysql 2>/dev/null)" ]; then
if ! /etc/init.d/mysql status; then can=$(date -r /mysql +%s)
echo "Starting mysql ..." now=$(date +%s)
/etc/init.d/mysql start age=$((($now - $can)/86400))
echo "NOTE: Database empty; populating it from canned snapshot ($age days old)"
echo " This will take a little while..."
cp -r /mysql/* $MYSQLDIR
fi fi
echo "Checking if syslogd is running ..." service mariadb start
if ! /etc/init.d/rsyslog status > /dev/null; then
echo "Starting syslogd ..." if ! service mariadb status; then
/etc/init.d/rsyslog start echo "ERROR: MySQL didn't start. Here are some possible causes:"
echo "-------------------------------------------------------------------"
grep mysqld /var/log/syslog
echo "-------------------------------------------------------------------"
echo "Such errors are usually due to a corrupt or outdated database."
echo "Remove your local database and let the image install a clean copy."
exit 1
fi fi
# Give debian-sys-maint access, to avoid complaints later if [ ! -f /root/src/ietf/settings_local.py ]; then
mysql mysql <<< "GRANT ALL PRIVILEGES on *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '$(awk '/^password/ {print $3; exit}' /etc/mysql/debian.cnf )' WITH GRANT OPTION; FLUSH PRIVILEGES;"
echo "Checking if the IETF database exists at $MYSQLDIR ..."
if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
if [ -z "$DATADIR" ]; then
echo "DATADIR is not set, but the IETF database needs to be set up -- can't continue, exiting the docker init script."
exit 1
fi
ls -l $MYSQLDIR
if ! /etc/init.d/mysql status; then
echo "Didn't find the IETF database, but can't set it up either, as MySQL isn't running."
else
echo "Creating database ..."
mysqladmin -u root --default-character-set=utf8 create ietf_utf8
echo "Setting up permissions ..."
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
echo "Fetching database ..."
DUMPDIR=/home/$USER/$DATADIR
wget -N -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz
echo "Loading database ..."
gunzip < $DUMPDIR/ietf_utf8.sql.gz \
| pv --progress --bytes --rate --eta --cursor --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| mysql --user=django --password=RkTkDPFnKpko -s -f ietf_utf8 \
&& rm /tmp/ietf_utf8.sql.gz
fi
fi
if ! grep -q ":$GID:$" /etc/group ; then
echo "Creating group entry for GID '$GID' ..."
groupadd -g "$GID" "$USER"
fi
if ! id -u "$USER" &> /dev/null; then
echo "Creating user '$USER' ..."
useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER
echo "$USER:$USER" | chpasswd
fi
VIRTDIR="/opt/home/$USER/$TAG"
echo "Checking that there's a virtual environment for $TAG ..."
if [ ! -f $VIRTDIR/bin/activate ]; then
echo "Setting up python virtualenv at $VIRTDIR ..."
mkdir -p $VIRTDIR
python3.6 -m venv $VIRTDIR
echo -e "
# This is from $VIRTDIR/bin/activate, to activate the
# datatracker virtual python environment on docker container entry:
" >> /etc/bash.bashrc
cat $VIRTDIR/bin/activate >> /etc/bash.bashrc
cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc
else
echo "Using virtual environment at $VIRTDIR"
fi
echo "Activating the virtual python environment ..."
. $VIRTDIR/bin/activate
if ! $VIRTDIR/bin/python -c "import django"; then
echo "Installing requirements ..."
pip install --upgrade pip
reqs=/home/$USER/$CWD/requirements.txt
if [ ! -f $reqs ]; then
echo " Using $reqs"
pip install -r $reqs
else
echo " Didn't find $reqs"
echo " Using /usr/local/share/datatracker/requirements.txt"
pip install -r /usr/local/share/datatracker/requirements.txt
fi
fi
if [ ! -f /home/$USER/$CWD/ietf/settings_local.py ]; then
echo "Setting up a default settings_local.py ..." echo "Setting up a default settings_local.py ..."
cp /home/$USER/$CWD/docker/settings_local.py /home/$USER/$CWD/ietf/settings_local.py cp /root/src/docker/settings_local.py /root/src/ietf/settings_local.py
fi fi
for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do
dir="/home/$USER/$CWD/$sub"
if [ ! -d "$dir" ]; then
echo "Creating dir $dir"
mkdir -p "$dir";
fi
done
for sub in \ for sub in \
nomcom_keys/public_keys \ test/id \
developers/ietf-ftp \ test/staging \
developers/ietf-ftp/bofreq \ test/archive \
developers/ietf-ftp/charter \ test/rfc \
developers/ietf-ftp/conflict-reviews \ test/media \
developers/ietf-ftp/internet-drafts \ test/wiki/ietf \
developers/ietf-ftp/rfc \ data/nomcom_keys/public_keys \
developers/ietf-ftp/status-changes \ data/developers/ietf-ftp \
developers/ietf-ftp/yang/catalogmod \ data/developers/ietf-ftp/bofreq \
developers/ietf-ftp/yang/draftmod \ data/developers/ietf-ftp/charter \
developers/ietf-ftp/yang/ianamod \ data/developers/ietf-ftp/conflict-reviews \
developers/ietf-ftp/yang/invalmod \ data/developers/ietf-ftp/internet-drafts \
developers/ietf-ftp/yang/rfcmod \ data/developers/ietf-ftp/rfc \
developers/www6s \ data/developers/ietf-ftp/status-changes \
developers/www6s/staging \ data/developers/ietf-ftp/yang/catalogmod \
developers/www6s/wg-descriptions \ data/developers/ietf-ftp/yang/draftmod \
developers/www6s/proceedings \ data/developers/ietf-ftp/yang/ianamod \
developers/www6/ \ data/developers/ietf-ftp/yang/invalmod \
developers/www6/iesg \ data/developers/ietf-ftp/yang/rfcmod \
developers/www6/iesg/evaluation \ data/developers/www6s \
data/developers/www6s/staging \
data/developers/www6s/wg-descriptions \
data/developers/www6s/proceedings \
data/developers/www6/ \
data/developers/www6/iesg \
data/developers/www6/iesg/evaluation \
; do ; do
dir="/home/$USER/$CWD/data/$sub" dir="/root/src/$sub"
if [ ! -d "$dir" ]; then if [ ! -d "$dir" ]; then
echo "Creating dir $dir" echo "Creating dir $dir"
mkdir -p "$dir"; mkdir -p "$dir";
chown "$USER" "$dir"
fi fi
done done
if [ ! -f "/home/$USER/$CWD/test/data/draft-aliases" ]; then python -m smtpd -n -c DebuggingServer localhost:2025 &
echo "Generating draft aliases ..." echo
ietf/bin/generate-draft-aliases }
fi
if [ ! -f "/home/$USER/$CWD/test/data/group-aliases" ]; then if [ -z "$*" ]; then
echo "Generating group aliases ..." echo "You can execute arbitrary commands now, e.g.,"
ietf/bin/generate-wg-aliases } echo
fi echo " ietf/manage.py runserver 0.0.0.0:8000"
echo
chown -R $USER /opt/home/$USER echo "to start a development instance of the Datatracker."
chmod -R g+w /usr/local/lib/ # so we can patch libs if needed echo
bash
cd "/home/$USER/$CWD" || cd "/home/$USER/"
echo "Done!"
if ! echo "$LANG" | grep "UTF-8"; then
echo ""
echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc"
else else
echo "LANG=$LANG" echo "Executing \"$*\" and stopping container."
echo
bash -c "$*"
fi fi
HOME=/opt/home/$USER service mariadb stop
service rsyslog stop
su -p $USER

14
docker/install-extras
View file

@ -1,14 +0,0 @@
#!/bin/bash
export DEBIAN_FRONTEND=noninteractive
sudo su - -c "apt-get update \
&& apt-get install -qy \
apt-file \
apt-show-versions \
graphviz \
ghostscript \
apache2-utils \
chromium-driver \
enscript \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*"

13
docker/mailserver
View file

@ -1,13 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
$progdir/run python -m smtpd -n -c DebuggingServer localhost:2025

102
docker/rsync-extras
View file

@ -1,127 +1,105 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.11
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
export LANG=C
# ----------------------------------------------------------------------
function usage() { function usage() {
cat <<EOF cat <<EOF
NAME NAME
$program - Copy additional data files from the ietf server $program - Copy additional data files from the ietf server
SYNOPSIS SYNOPSIS
$program [OPTIONS] $program [OPTIONS] [DESTINATION]
DESCRIPTION DESCRIPTION
This script copies additional data files used by the datatracker This script copies additional data files used by the datatracker
from the ietf server to a local directory, for instance drafts, from the ietf server to a local directory, for instance drafts,
charters, rfcs, agendas, minutes, etc. charters, rfcs, agendas, minutes, etc.
If no destination is given, the default is data/developers.
OPTIONS
EOF EOF
echo -e "OPTIONS" grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF cat <<EOF
FILES
AUTHOR AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com> Written by:
Henrik Levkowetz, <henrik@levkowetz.com>
Lars Eggert, <lars@eggert.org>
COPYRIGHT COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of Copyright (c) 2016 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified the code. All rights reserved. Redistribution and use in source and
in http://opensource.org/licenses/BSD-3-Clause. binary forms, with or without modification, is permitted pursuant to,
and subject to the license terms contained in, the Revised BSD
EOF License set forth in Section 4.c of the IETF Trust’s Legal Provisions
Relating to IETF Documents(https://trustee.ietf.org/license-info).
EOF
} }
# ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
} }
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing # Option parsing
# Options
shortopts=hvV shortopts=hvV
longopts=help,verbose,version args=$(getopt -o$shortopts $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
# Default values set -- $args
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do while true ; do
case "$1" in case "$1" in
-h| --help) usage; exit;; # Show this help, then exit -h) usage; exit;; # Show this help, then exit
-v| --verbose) VERBOSE=1;; # Be more talkative -v) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit -V) version; exit;; # Show program version, then exit
--) shift; break;; --) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";; *) die "Internal error, inconsistent option specification: '$1'";;
esac esac
shift shift
done done
# ----------------------------------------------------------------------
# The program itself # The program itself
if [ $# -lt 1 ]; then
DEST_ROOT=data/developers
else
DEST_ROOT="${1%/}"
fi
echo "Using destination $DEST_ROOT"
[ $# -lt 1 ] && die "Missing argument: rsync destination"
DEST_ROOT="${1%/}"
for dir in charter conflict-reviews internet-drafts review rfc slides status-changes yang; do for dir in charter conflict-reviews internet-drafts review rfc slides status-changes yang; do
dest="$DEST_ROOT/ietf-ftp/$dir" dest="$DEST_ROOT/ietf-ftp/$dir"
mkdir -p "$dest"
echo "Fetching $dest ..." echo "Fetching $dest ..."
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::everything-ftp/$dir/ $dest/ rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::everything-ftp/$dir/ $dest/
done done
for dir in floor photo; do for dir in floor photo; do
dest="$DEST_ROOT/media/$dir" dest="$DEST_ROOT/media/$dir"
mkdir -p "$dest"
echo "Fetching $dest ..." echo "Fetching $dest ..."
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::dev.media/$dir/ $dest/ rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::dev.media/$dir/ $dest/
done done
dest="$DEST_ROOT/archive/id"
dest="$DEST_ROOT/archive/id" mkdir -p "$dest"
echo "Fetching $dest ..." echo "Fetching $dest ..."
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::id-archive/ $dest/ rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::id-archive/ $dest/

277
docker/run
View file

@ -1,15 +1,15 @@
#!/bin/bash #!/bin/bash
version=0.10 version=0.20
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo $parent | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
# ----------------------------------------------------------------------
function usage() { function usage() {
cat <<EOF cat <<EOF
NAME NAME
@ -19,269 +19,92 @@ SYNOPSIS
$program [OPTIONS] ARGS $program [OPTIONS] ARGS
DESCRIPTION DESCRIPTION
This is a wrapper which runs an Ubuntu-based docker image which
has been set up with the dependencies needed to easily run the
IETF datatracker in development mode.
This is a wrapper which runs docker with suitable arguments on a MySQL database files at data/mysql will be used; if they do not exist,
debian-based docker image which has been set up with the dependencies a database dump will be retrieved and restored on first run.
needed to easily run the IETF datatracker in development mode. By
default, it expects to find the MySQL database files at
$parent/data/mysql, which is mapped inside the
container to /var/lib/mysql, and it will set up a home directory for
the current user ($USER) and map it to $HOME.
OPTIONS
EOF EOF
echo -e "OPTIONS" grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF cat <<EOF
FILES
AUTHOR AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com> Written by:
Henrik Levkowetz, <henrik@levkowetz.com>
Lars Eggert, <lars@eggert.org>
COPYRIGHT COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of
Copyright (c) 2015 IETF Trust and the persons identified as authors of the code. All rights reserved. Redistribution and use in source and
the code. All rights reserved. License 'Simplified BSD', as specified binary forms, with or without modification, is permitted pursuant to,
in http://opensource.org/licenses/BSD-3-Clause. and subject to the license terms contained in, the Revised BSD
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
Relating to IETF Documents(https://trustee.ietf.org/license-info).
EOF EOF
} }
# ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
} }
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=dhi:m:Mp:r:t:vVu:cC
longopts=download-data,help,ietfdb-url=,mysqldata=,no-mysqldir,port=,docker-repo=,tag=,verbose,version,user=,cached,no-cached
# Default values # Default values
MYSQLDIR=$parent/data/mysql MYSQLDIR=$parent/data/mysql
FILEDIR=$parent/../data/
PORT=8000 PORT=8000
REPO="ietf/datatracker-environment" REPO="ietf/datatracker-environment"
DBURL=https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2 CACHED=':cached'
WHO=$(whoami)
CACHED=''
if [ "$(uname)" = "Linux" ]; then # Option parsing
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") shortopts=cChp:V
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi args=$(getopt -o$shortopts $*)
eval set -- "$args" if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
sed="sed -r" set -- $args
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
if [ "$(uname)" = "Darwin" ]; then
CACHED=':cached'
fi
fi
while true ; do while true ; do
case "$1" in case "$1" in
-c| --cached) CACHED=':cached';; # Use cached disk access to reduce system load -c) CACHED=':cached';; # Use cached disk access to reduce system load
-C| --no-cached) CACHED='';; # Use fully synchronized disk access -C) CACHED=':consistent';; # Use fully synchronized disk access
-d| --download-data) DOWNLOAD=1;; # Download and set up the database files -h) usage; exit;; # Show this help, then exit
-h| --help) usage; exit;; # Show this help, then exit -p) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
-f| --filedir) FILEDIR=$2; shift;; # Set the desired location of drafts, charters etc. -V) version; exit;; # Show program version, then exit
-i| --ietfdb-url) DBURL=$2; shift;; # Use an alternative database tarball URL --) shift; break;;
-m| --mysqldir) MYSQLDIR=$2; shift;; # Set the desired location of MySQL's database files *) die "Internal error, inconsistent option specification: '$1'";;
-p| --port) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
-r| --docker-repo) REPO=$2; shift;; # Use the given docker repository, instead of the default
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the svn branch name
-u| --user) WHO=$2; shift;; # Run the container as the specified user
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit
--) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";;
esac esac
shift shift
done done
if [ -z "$TAG" ]; then if [ -z "$TAG" ]; then
TAG=$(basename $(svn info $parent | grep ^URL | awk '{print $2}')) TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}' | tr -d '\r')")
fi fi
# ---------------------------------------------------------------------- if [[ $(uname) =~ CYGWIN.* ]]; then
# The program itself echo "Running under Cygwin, replacing symlinks with file copies"
ICSFILES=$(/usr/bin/find "$parent/vzic/zoneinfo/" -name '*.ics' -print)
[ -f /proc/1/cgroups ] && grep 'docker' /proc/1/cgroups && die "It looks like you're inside docker already ..." for ICSFILE in $ICSFILES; do
LINK=$(head -n1 "$ICSFILE" | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
if [ "$(uname)" = "Darwin" ]; then
APP="/Applications/Docker.app"
CMD="open -a"
elif [ "$(uname)" = "Linux" ]; then
echo "Running on Linux."
elif [[ $(uname) =~ CYGWIN.* ]]; then
echo "Running under Cygwin."
CMD="echo"
MYSQLDIR=$(echo $MYSQLDIR | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
WHO=$(echo $WHO | sed -e 's/^.*\\//' | tr -d \\r)
DRIVE=$(echo $USERPROFILE | sed -e 's/\(.\).*/\1/' -e 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefdhijklmnopqartuvwxyz/')
HOME=$DRIVE$(echo $USERPROFILE | sed -e 's/\\/\//g' -e's/.\(.*\)/\1/')
echo "Using home dir $HOME"
PWD=$(echo $PWD | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
if [ "${PWD#$HOME}" = "$PWD" ]; then
die "You must work inside your home directory ($HOME)"
fi
CGWPARENT=$(echo $parent | sed -e 's/^\(.\)\:/\/cygdrive\/\1/')
ICSFILES=$(/usr/bin/find $CGWPARENT/vzic/zoneinfo/ -name '*.ics' -print)
for ICSFILE in $ICSFILES
do
LINK=$(head -n1 $ICSFILE | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
if [ "$LINK" ]; then if [ "$LINK" ]; then
WDIR=$(dirname $ICSFILE) WDIR=$(dirname "$ICSFILE")
echo "Replacing $(basename $ICSFILE) with $LINK" echo "Replacing $(basename "$ICSFILE") with $LINK"
cp -f $WDIR/$LINK $ICSFILE cp -f "$WDIR/$LINK" "$ICSFILE"
fi fi
done done
else
die "This script does not have support for your architecture ($(uname)); sorry :-("
fi fi
WHOUID=$(id -u $WHO) echo "Starting a docker container for '$REPO:$TAG'."
WHOGID=$(id -g $WHO) mkdir -p "$MYSQLDIR"
docker run -ti -p "$PORT":8000 -p 33306:3306 \
if [ "$(uname)" = "Linux" ]; then -v "$parent:/root/src$CACHED" \
echo "Not trying to start a virtual docker machine on Linux" -v "$MYSQLDIR:/var/lib/mysql:delegated" \
elif [[ $(uname) =~ CYGWIN.* ]]; then "$REPO:$TAG" "$@"
if ! docker info 1> /dev/null 2>&1; then
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
die "Don't know how to start docker when running under Cygwin"
fi
TAG=$(echo $TAG | tr -d \\r)
URL="http://localhost:$PORT/"
elif [ -e "$APP" ]; then
info=$(docker info 2>&1 || true)
if ! docker info 1> /dev/null 2>&1; then
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
echo "Will attempt to start docker by doing '\$ $CMD $APP':"
$CMD $APP
declare -i count
printf "Waiting for docker engine .."
while true; do
printf "."
sleep 2
if docker info >/dev/null 2>&1; then break; fi
count=$(( $count + 1))
if [ $count -gt 10 ]; then
die "No contact with docker engine; bailing out."
fi
done
fi
URL="http://localhost:$PORT/"
else
if [ $($machine status default | tr "A-Z" "a-z" ) != "running" ]; then
echo "The docker VM doesn't seem to be running; will attempt to start it by doing '\$ $machine start':"
$machine start || die "Failed taking up the Docker VM"
fi
if [ -f "$machine" ]; then
if [ $($machine status default | tr "A-Z" "a-z") = "running" ]; then
tmpfile=$(mktemp docker.run.XXXXXXXX)
if $machine env 2>/dev/null | grep DOCKER_ > $tmpfile; then
mv $tmpfile ~/.docker-info
elif printenv | grep DOCKER_ > $tmpfile; then
mv $tmpfile ~/.docker-info
else
rm $tmpfile
die "Failed setting the appropriate DOCKER_* environment variables."
fi
. ~/.docker-info
else
rm -f ~/.docker-info
fi
ip=$($machine ip)
URL="http://$ip:$PORT/"
fi
fi
image=$(docker ps | grep "$REPO:$TAG" | awk '{ print $1 }')
if [ "$image" ]; then
if [ "$*" ]; then
echo "Running 'cd ~/${parent#$HOME/}; $*'"
docker exec -u "$WHO" -t $image bash -i -c "cd; cd \"${parent#$HOME/}\"; $*"
else
docker exec -u "$WHO" -ti $image bash -i
fi
else
echo ""
echo "Starting a docker container for '$TAG'."
if [ -n "$DOWNLOAD" ]; then
(
cd "$(dirname $MYSQLDIR)"
wget -N "$DBURL" && tar xjf ietf_utf8.bin.tar.bz2 && chmod -R go+rwX mysql
)
[ -d "$MYSQLDIR" ] || die "The download seems to have failed; still no $MYSQLDIR. Giving up."
else
[ -d "$MYSQLDIR" ] || die "Expected $MYSQLDIR to exist, but it\ndidn't. Use '$program -d' to download and unpack the database."
fi
if ! docker images "$REPO" | grep -q "$TAG"; then
echo "Fetching docker image '$REPO:$TAG'"
if ! docker pull "$REPO:$TAG"; then
docker pull "$REPO:latest" || die "Failed to pull down the '$REPO:latest' docker image"
id=$(docker images "$REPO" | grep latest | awk '{print $3}')
echo "Tagging $REPO:latest as $REPO:$TAG for use as environment for this branch."
docker tag $id "$REPO:$TAG"
fi
fi
echo -e "\nThe web interface for 'runserver' should appear on $URL\n"
echo -e "User $WHO ($WHOUID:$WHOGID)"
if [ -z "$MYSQLDIR" ]; then
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
-e USER="$WHO" -e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" \
-e TAG="$TAG" -e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
"$REPO:$TAG" "$@"
else
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
-v "$MYSQLDIR:/var/lib/mysql" -e USER="$WHO" \
-e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" -e TAG="$TAG" \
-e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
"$REPO:$TAG" "$@"
fi
echo ""
echo "Committing changes in the container to an image:"
image=$( docker images -q $REPO:$TAG)
latest=$(docker ps -lq -f "ancestor=$image")
docker commit $latest $REPO:$TAG
echo ""
echo "Cleaning up containers and images"
docker rm $latest
DANGLING=$( docker images -f dangling=true -q )
if [ -n "$DANGLING" ]; then
echo "Dangling images: $DANGLING"
docker rmi -f $DANGLING
fi
fi

26
docker/setprompt
View file

@ -1,26 +0,0 @@
# set a fancy prompt
PS_TIME="\A "
if [ "$(whoami)" = "root" ]; then
PS_COLOUR='\[\033[1;31m\]'
PS_END="\[\033[m\]"
else
PS_COLOUR="\[\033[1;34m\]"
PS_END="\[\033[m\]"
fi
if [ "$TERM" = xterm ]; then
PS_XTERM="\[\033]0;\]\h:\w\007"
else
PS_XTERM=""
fi
COLOUR_RED='\[\033[1;31m\]'
COLOUR_BLK='\[\033[1;30m\]'
if [ "$SHLVL" -gt 1 ]; then
PS_SHLVL="$(eval "printf '>%.0s' {2..$SHLVL}") "
else
PS_SHLVL=""
fi
PS1="$PS_XTERM\n$COLOUR_RED$PS_SHLVL$PS_COLOUR$PS_TIME$COLOUR_RED$PS_COLOUR${VIRTUAL_ENV:+$COLOUR_RED($(basename $VIRTUAL_ENV))$PS_COLOUR }\w\n\u @ $COLOUR_BLK\h$PS_COLOUR \\$ $PS_END"

114
docker/setupdb
View file

@ -1,114 +0,0 @@
#!/bin/bash
version=0.10
program=${0##*/}
progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir)
if [ "$parent" = "." ]; then parent="$PWD"; fi
# ----------------------------------------------------------------------
function usage() {
cat <<EOF
NAME
$program - Set up a local copy of the IETF database MySQL files
SYNOPSIS
$program [OPTIONS] ARGS
DESCRIPTION
This script downloads a prebuilt copy of the IETF database MySQL
files, ready for mapping into the /var/lib/mysql/ directory of the
ietf/database-environment Docker image.
EOF
echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF
FILES
AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com>
COPYRIGHT
Copyright (c) 2015 IETF Trust and the persons identified as authors of
the code. All rights reserved. License 'Simplified BSD', as specified
in http://opensource.org/licenses/BSD-3-Clause.
EOF
}
# ----------------------------------------------------------------------
function die() {
echo -e "\n$program: error: $*" >&2
exit 1
}
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() {
echo -e "$program $version"
}
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=hvV
longopts=help,verbose,version
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do
case "$1" in
-h| --help) usage; exit;; # Show this help, then exit
-v| --verbose) VERBOSE=1;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit
--) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";;
esac
shift
done
# ----------------------------------------------------------------------
# The program itself
[ -n "$MYSQLDIR" ] || MYSQLDIR=$parent/data/mysql
[ -n "$URL" ] || URL=rsync.ietf.org::dev.db/ietf_utf8.bin.tar.bz2
cd $(dirname $MYSQLDIR)
echo "Downloading a MySQL database image ..."
set $(rsync --version | head -n1 | awk '{print $3}' | tr '.' ' ')
maj=$1; min=$2; fix=$3
if [ "$maj" -ge "3" -a "$min" -ge "1" ]; then
rsync --info=progress2 $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
else
rsync -v $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
fi

137
docker/updatedb
View file

@ -1,15 +1,15 @@
#!/bin/bash #!/bin/bash
# -*- indent-with-tabs: 1 -*-
version=0.10 version=0.20
program=${0##*/} program=${0##*/}
progdir=${0%/*} progdir=${0%/*}
if [ "$progdir" = "$program" ]; then progdir="."; fi if [ "$progdir" = "$program" ]; then progdir="."; fi
if [ "$progdir" = "." ]; then progdir="$PWD"; fi if [ "$progdir" = "." ]; then progdir="$PWD"; fi
parent=$(dirname $progdir) parent=$(dirname "$progdir")
if [ "$parent" = "." ]; then parent="$PWD"; fi if [ "$parent" = "." ]; then parent="$PWD"; fi
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
# ----------------------------------------------------------------------
function usage() { function usage() {
cat <<EOF cat <<EOF
NAME NAME
@ -19,140 +19,87 @@ SYNOPSIS
$program [OPTIONS] ARGS $program [OPTIONS] ARGS
DESCRIPTION DESCRIPTION
This script downloads a dump of the IETF database and loads into the This script downloads a dump of the IETF database and loads into the
local sql server if it is newer than the current dump. local sql server if it is newer than the current dump.
OPTIONS
EOF EOF
echo -e "OPTIONS" grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF cat <<EOF
FILES
AUTHOR AUTHOR
Written by Henrik Levkowetz, <henrik@levkowetz.com> Written by:
Henrik Levkowetz, <henrik@levkowetz.com>
Lars Eggert, <lars@eggert.org>
COPYRIGHT COPYRIGHT
Copyright (c) 2016 IETF Trust and the persons identified as authors of
Copyright (c) 2015 IETF Trust and the persons identified as authors of the code. All rights reserved. Redistribution and use in source and
the code. All rights reserved. License 'Simplified BSD', as specified binary forms, with or without modification, is permitted pursuant to,
in http://opensource.org/licenses/BSD-3-Clause. and subject to the license terms contained in, the Revised BSD
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
Relating to IETF Documents(https://trustee.ietf.org/license-info).
EOF EOF
} }
# ----------------------------------------------------------------------
function die() { function die() {
echo -e "\n$program: error: $*" >&2 echo -e "\n$program: error: $*" >&2
exit 1 exit 1
} }
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
# ----------------------------------------------------------------------
function version() { function version() {
echo -e "$program $version" echo -e "$program $version"
} }
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing # Option parsing
shortopts=DLZhV
# Options
shortopts=DLZhqvV
longopts=no-download,no-load,no-zap,help,quiet,verbose,version
LOAD=1 LOAD=1
DOWNLOAD=1 DOWNLOAD=1
DROP=1 DROP=1
QUIET=""
if [ "$(uname)" = "Linux" ]; then args=$(getopt -o$shortopts $*)
args=$(getopt -o "$shortopts" --long "$longopts" -n "$program" -- $SV "$@") if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi set -- $args
eval set -- "$args"
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
set -- $args
sed="sed -E"
fi
while true ; do while true ; do
case "$1" in case "$1" in
-D| --no-download) DOWNLOAD="";; #Don't download, use existing file -D) DOWNLOAD="";; # Don't download, use existing file
-L| --no-load) LOAD=""; ;; # Don't load the database -L) LOAD=""; ;; # Don't load the database
-Z| --no-zap) DROP="";; # Don't drop new tables -Z) DROP="";; # Don't drop new tables
-h| --help) usage; exit;; # Show this help, then exit -h) usage; exit;; # Show this help, then exit
-q| --quiet) QUIET="-q";; # Don't show progress -V) version; exit;; # Show program version, then exit
-v| --verbose) VERBOSE=1;; # Be more talkative --) shift; break;;
-V| --version) version; exit;; # Show program version, then exit *) die "Internal error, inconsistent option specification: '$1'";;
--) shift; break;;
*) die "Internal error, inconsistent option specification: '$1'";;
esac esac
shift shift
done done
# ----------------------------------------------------------------------
# The program itself # The program itself
echo "Gathering info ..."
MYSQLDIR="$(/usr/sbin/mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
MYSQLDIR=${MYSQLDIR%/}
DATADIR=$parent/data DATADIR=$parent/data
DUMP=ietf_utf8.sql.gz
# echo "Checking if MySQL base data exists ..."
# if [ ! -d $MYSQLDIR/mysql ]; then
# die "Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
# fi
# echo "Checking if the IETF database exists at $MYSQLDIR ..."
# if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
# echo "Creating database ..."
# mysqladmin -u root --default-character-set=utf8 create ietf_utf8
#
# echo "Setting up permissions ..."
# mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
# fi
if [ "$DOWNLOAD" ]; then if [ "$DOWNLOAD" ]; then
echo "Fetching database dump ..." echo "Fetching database dump..."
wget $QUIET -N -P $DATADIR https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz || die "No new data, quitting." rsync --info=progress2 rsync.ietf.org::dev.db/$DUMP "$DATADIR"
fi fi
if [ "$LOAD" ]; then if [ "$LOAD" ]; then
echo "Loading database ..." echo "Loading database..."
if [ -z "$QUIET" ]; then SIZE=$(pigz --list "$DATADIR/$DUMP" | tail -n 1 | awk '{ print $2 }')
gunzip < $DATADIR/ietf_utf8.sql.gz \ pigz -d < "$DATADIR/$DUMP" \
| pv --progress --bytes --rate --eta --size $(gzip --list --quiet $DATADIR/ietf_utf8.sql.gz | awk '{ print $2 }') \ | pv --progress --bytes --rate --eta --size "$SIZE" \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \ | sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| $parent/ietf/manage.py dbshell | "$parent/ietf/manage.py" dbshell
else
gunzip < $DATADIR/ietf_utf8.sql.gz \
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
| $parent/ietf/manage.py dbshell
fi
fi fi
if [ "$DROP" ]; then if [ "$DROP" ]; then
echo "Dropping tables not in the dump (so migrations can succeed) ..." echo "Dropping tables not in the dump (so migrations can succeed)..."
diff <(zcat $DATADIR/ietf_utf8.sql.gz | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \ diff <(pigz -d -c "$DATADIR/$DUMP" | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
<($parent/ietf/manage.py dbshell <<< 'show tables;' | tail -n +2) \ <("$parent/ietf/manage.py" dbshell <<< 'show tables;' | tail -n +2) \
| grep '^>' | awk '{print "drop table if exists", $2, ";";}' \ | grep '^>' | awk '{print "drop table if exists", $2, ";";}' \
| tee >(cat >&2) | $parent/ietf/manage.py dbshell | tee >(cat >&2) | "$parent/ietf/manage.py" dbshell
fi fi