Version of the new docker setup emailed out for testing on 2021-10-12
- Legacy-Id: 19417
This commit is contained in:
Lars Eggert
parent
3386e59a61
commit
6fb64998ce
|
|
@ -1,185 +1,85 @@
|
|||
# -*- shell-mode -*-
|
||||
# This file is a docker (https://www.docker.com/what-docker) recipe, which can be used to build
|
||||
# a docker image which is ready to run a datatracker in development mode.
|
||||
# This is a Dockerfile with everything in it to run the IETF datatracker.
|
||||
#
|
||||
# It is used to build an image (once you've installed docker) using a command like this (assuming
|
||||
# suitable replacement of $variables:
|
||||
# If you make changes to the datatracker that add new dependencies (python
|
||||
# packages or otherwise), you need to rebuild this image to make them
|
||||
# available. Do this in the top-level directory of your datatracker source
|
||||
# tree:
|
||||
#
|
||||
# $ docker build -t $yourdockerid/datatracker:$version
|
||||
# docker build --tag dt:latest .
|
||||
#
|
||||
# To use a pre-built image, assuming we're on OS X and have a checked-out datatracker repository
|
||||
# at /Users/$login/src/6.8.1.dev0, you would start (again assuming you've installed docker)
|
||||
# a container from an image, as follows:
|
||||
#
|
||||
# $ docker run -ti --name=$containername -v /Users/$login:/home/$login levkowetz/datatracker:$version /bin/bash
|
||||
#
|
||||
# This maps your home directory to /home/$login in the container, and starts it running /bin/bash.
|
||||
#
|
||||
# In this first version, the docker environment is set up so that tests will run successfully,
|
||||
# but the database has *not* been loaded with a dump, and supporting files (drafts, charters, etc.)
|
||||
# have *not* been downloaded.
|
||||
# You can then execute the datatracker like this (also from the top-level
|
||||
# datatracker source directory):
|
||||
#
|
||||
# docker run -ti -v $(pwd):/root/src dt:latest
|
||||
|
||||
FROM dyne/devuan:beowulf
|
||||
LABEL maintainer="Henrik Levkowetz <henrik@levkowetz.com>"
|
||||
FROM ubuntu:hirsute
|
||||
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
|
||||
|
||||
# Default django runserver port
|
||||
EXPOSE 8000
|
||||
EXPOSE 8000
|
||||
|
||||
# Run apt-get noninteractive
|
||||
# Default mysqld/mariadb port
|
||||
EXPOSE 3306
|
||||
|
||||
# Install all dependencies that are available as packages
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV DEVUAN_FRONTEND=noninteractive
|
||||
|
||||
# Uncomment this to be able to install and run apt-show-versions:
|
||||
RUN rm -v /etc/apt/apt.conf.d/docker-compress
|
||||
RUN rm -v /var/lib/apt/lists/*lz4
|
||||
|
||||
RUN apt-get update --allow-releaseinfo-change
|
||||
RUN apt-get install -qy apt-transport-https
|
||||
|
||||
# Use backports, updates, and security updates
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf main contrib non-free" > /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-security main contrib non-free" >> /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-updates main contrib non-free" >> /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-backports main contrib non-free" >> /etc/apt/sources.list
|
||||
|
||||
# Remove some excludes for the docker image
|
||||
RUN sed -i -e '/^path-exclude=.*\/groff/d' \
|
||||
-e '/^path-exclude=.*\/locale/d' \
|
||||
-e '/^path-exclude=.*\/man/d' /etc/dpkg/dpkg.cfg.d/docker-excludes
|
||||
|
||||
# Install needed packages
|
||||
#
|
||||
# We're not including graphviz and ghostscript, needed for the 3 document
|
||||
# dependency graph tests; they would increase the size of the image by about
|
||||
# 15%, about 100MB.
|
||||
|
||||
# Fetch apt package information, and upgrade to latest package versions
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get -qy upgrade
|
||||
|
||||
# Install the packages we need
|
||||
RUN apt-get install -qy \
|
||||
build-essential \
|
||||
bzip2 \
|
||||
ca-certificates \
|
||||
colordiff \
|
||||
gawk \
|
||||
gcc \
|
||||
ipython \
|
||||
jq \
|
||||
less \
|
||||
libbz2-dev \
|
||||
libdb5.3-dev \
|
||||
libexpat1-dev \
|
||||
libffi-dev \
|
||||
libgdbm-dev \
|
||||
libjpeg62-turbo-dev \
|
||||
liblzma-dev \
|
||||
libmagic1 \
|
||||
libmariadbclient-dev \
|
||||
libncurses5-dev \
|
||||
libncursesw5-dev \
|
||||
libreadline-dev \
|
||||
libsqlite3-dev \
|
||||
libssl-dev \
|
||||
libsvn1 \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
libz-dev \
|
||||
libffi-dev \
|
||||
locales \
|
||||
make \
|
||||
man \
|
||||
mariadb-client \
|
||||
mariadb-server \
|
||||
openssh-client \
|
||||
patch \
|
||||
procps \
|
||||
pv \
|
||||
rsync \
|
||||
RUN apt-get -y update && \
|
||||
apt-get -y install --no-install-recommends \
|
||||
apache2-utils \
|
||||
apt-file \
|
||||
apt-utils \
|
||||
chromium-driver \
|
||||
curl \
|
||||
enscript \
|
||||
gcc \
|
||||
ghostscript \
|
||||
git \
|
||||
graphviz \
|
||||
libmagic-dev \
|
||||
libmariadb-dev \
|
||||
mariadb-server \
|
||||
npm \
|
||||
pigz \
|
||||
pv \
|
||||
python-is-python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
rsyslog \
|
||||
subversion \
|
||||
sudo \
|
||||
uuid-dev \
|
||||
vim \
|
||||
wget \
|
||||
xz-utils\
|
||||
zile \
|
||||
zlib1g-dev
|
||||
unzip \
|
||||
yang-tools && \
|
||||
apt-get -y clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Postgresql packages
|
||||
RUN apt-get install -qy \
|
||||
postgresql-11 \
|
||||
postgresql-server-dev-11
|
||||
# Install bower
|
||||
RUN npm install -g bower
|
||||
|
||||
# Get the key used to sign the libyang repo
|
||||
RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key
|
||||
RUN apt-key add - < Release.key
|
||||
RUN rm Release.key
|
||||
|
||||
# Add apt source entry for libyang
|
||||
RUN echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list
|
||||
|
||||
# Update the package defs, and install the desired mysql from the mysql repo
|
||||
RUN apt-get update
|
||||
RUN apt-get install -qy libyang1
|
||||
|
||||
# This is expected to exist by the mysql startup scripts:
|
||||
#RUN touch /etc/mysql/debian.cnf
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Get rid of installation files we don't need in the image, to reduce size
|
||||
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Enable some common locales
|
||||
RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen
|
||||
RUN locale-gen
|
||||
|
||||
# Remove an rsyslog module that we don't need, which also requires extra permissions
|
||||
RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf
|
||||
|
||||
# Set up root password
|
||||
RUN echo "root:root" | chpasswd
|
||||
|
||||
# MySQL
|
||||
VOLUME /var/lib/mysql
|
||||
|
||||
# idnits and dependencies
|
||||
# Install idnits
|
||||
ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/
|
||||
RUN chmod +rx /usr/local/bin/idnits
|
||||
|
||||
# Directory for Mac certs
|
||||
RUN mkdir /etc/certificates
|
||||
# Install current datatracker python dependencies
|
||||
COPY requirements.txt /
|
||||
RUN pip install -r /requirements.txt
|
||||
|
||||
# # Python 3
|
||||
# # Comment in if OS does not provide python3.6, which is the current
|
||||
# # production version
|
||||
ENV PYVER=3.6.10
|
||||
ENV PYREV=3.6
|
||||
# Turn off rsyslog kernel logging (doesn't work in Docker)
|
||||
RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
|
||||
|
||||
WORKDIR /usr/src
|
||||
RUN wget -q https://www.python.org/ftp/python/$PYVER/Python-$PYVER.tar.xz
|
||||
RUN tar xJf Python-$PYVER.tar.xz
|
||||
RUN rm Python-$PYVER.tar.xz
|
||||
WORKDIR /usr/src/Python-$PYVER/
|
||||
RUN ./configure
|
||||
RUN make
|
||||
RUN make altinstall
|
||||
WORKDIR /usr/src
|
||||
RUN rm -rf /usr/src/Python-$PYVER/
|
||||
# Allow access to mariadb over the network
|
||||
RUN sed -i 's/127.0.0.1/0.0.0.0/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
|
||||
ENV HOSTNAME="datatracker"
|
||||
# Turn on mariadb performance_schema
|
||||
RUN sed -i 's/\[mysqld\]/\[mysqld\]\nperformance_schema=ON/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
|
||||
ENV DDIR="/usr/local/share/datatracker"
|
||||
RUN mkdir -p $DDIR
|
||||
WORKDIR $DDIR
|
||||
|
||||
COPY requirements.txt ./
|
||||
COPY setprompt ./
|
||||
# Make the mariadb sys schema available for possible installation
|
||||
# We would normally use the next line, but that has a bug:
|
||||
# ADD https://github.com/FromDual/mariadb-sys/archive/master.zip /
|
||||
# This is the repo that has the PR:
|
||||
ADD https://github.com/grooverdan/mariadb-sys/archive/refs/heads/master.zip /
|
||||
RUN unzip /master.zip
|
||||
|
||||
# Copy the startup file
|
||||
COPY docker-init.sh /docker-init.sh
|
||||
RUN chmod +x /docker-init.sh
|
||||
ENTRYPOINT ["/docker-init.sh"]
|
||||
|
||||
CMD /bin/bash
|
||||
WORKDIR /root/src
|
||||
ENTRYPOINT ["/docker-init.sh"]
|
||||
37
docker/README.md
Normal file
37
docker/README.md
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Datatracker Development in Docker
|
||||
|
||||
1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred
|
||||
platform.
|
||||
|
||||
2. If you have a copy of the datatracker code checked out already, simply `cd`
|
||||
to the top-level directory.
|
||||
|
||||
If not, check out a datatracker branch as usual. We'll check out `trunk`
|
||||
below, but you can use any branch:
|
||||
|
||||
svn co https://svn.ietf.org/svn/tools/ietfdb/trunk
|
||||
cd trunk
|
||||
|
||||
3. **TEMPORARY:** Replace the contents of the `docker` directory with Lars'
|
||||
files.
|
||||
|
||||
4. **TEMPORARY:** Until Lars' changes have been merged and a docker image is
|
||||
available for download, you will need to build it locally:
|
||||
|
||||
docker/build -l
|
||||
|
||||
This will take a while, but only needs to be done once.
|
||||
|
||||
5. Use the `docker/run` script to start the datatracker container. You will be
|
||||
dropped into a shell from which you can start the datatracker and execute
|
||||
related commands as usual, for example:
|
||||
|
||||
ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
If you do not already have a copy of the IETF database available in the
|
||||
`data` directory, one will be downloaded and imported the first time you run
|
||||
`docker/run`. This will take some time.
|
||||
|
||||
Once the datatracker has started, you should be able to open
|
||||
[http://localhost:8000](http://localhost:8000) in a browser and see the
|
||||
landing page.
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
|
||||
==============================================================================
|
||||
Datatracker Development in a Docker Container (beta)
|
||||
==============================================================================
|
||||
|
||||
|
||||
Intro
|
||||
=====
|
||||
|
||||
Docker_ is a toolkit which lets you package software together with its
|
||||
dependencies in lightweight containers, and run it in isolated virtual
|
||||
environments.
|
||||
|
||||
During and just after IETF-94 I've spent quite a bit of time setting up a
|
||||
docker image which provides the dependencies needed to run the datatracker,
|
||||
and it's now available for beta testing. Hopefully this should make it
|
||||
substantially easier to get started with datatracker development.
|
||||
|
||||
Steps
|
||||
=====
|
||||
|
||||
1. Set up Docker on your preferred platform. Official installers exist for
|
||||
many Linux flavours, OS X, Windows and Cloud services. Here's the full `List
|
||||
of Installation Instructions`_.
|
||||
|
||||
Docker containers require the services of an underlying Linux API, which
|
||||
means that on OS X and Windows, these have to be provided by a virtual
|
||||
machine which runs a minimal Linux image. The virtual machine used on
|
||||
non-Linux platforms is commonly VirtualBox. On Linux kernels with version
|
||||
3.8 or later, no virtual machine is needed, as the docker images can be
|
||||
fully supported with the native kernel services.
|
||||
|
||||
Please follow the Docker installations all the way through to successfully
|
||||
running the ``hello-world`` example in a terminal window ( ``$ docker run
|
||||
hello-world``).
|
||||
|
||||
|
||||
2. Check out your datatracker branch as usual, in a suitable directory.
|
||||
We'll assume ``~/src/dt/`` here, and assume you are ``'coder'``::
|
||||
|
||||
~/src/dt/ $ svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/coder/6.8.2.dev0
|
||||
|
||||
3. In the checked-out working copy, you'll find a ``docker/`` directory and a
|
||||
``data/`` directory at the top level. We're first going to set up a copy of
|
||||
the MySQL database files under the ``data/`` directory.
|
||||
|
||||
There is a command in the ``docker/`` directory, ``setupdb`` which will do
|
||||
this for you, or you can do it manually.
|
||||
|
||||
Either run::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker/setupdb
|
||||
|
||||
or do this step-by-step: fetch down a pre-built copy of the datatracker
|
||||
database, place it in the ``data`` directory, unpack it, and fix
|
||||
permissions::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ cd data
|
||||
~/src/dt/6.8.2.dev0/data/ $ wget https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
|
||||
~/src/dt/6.8.2.dev0/data/ $ tar xjf ietf_utf8.bin.tar.bz2
|
||||
~/src/dt/6.8.2.dev0/data/ $ chmod -R go+rwX mysql
|
||||
|
||||
|
||||
4. In the ``docker/`` directory you'll also find a wrapper script named
|
||||
``'run'``. We will be using the wrapper to run a pre-built docker image
|
||||
fetched from the docker hub::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker/run
|
||||
|
||||
This will pull down the latest docker ietf/datatracker-environment image,
|
||||
start it up with appropriate settings, map the internal ``/var/lib/mysql/``
|
||||
directory to the external ``data/mysql/`` directory where we placed the
|
||||
database, set up a python virtualenv for you, install some dependencies,
|
||||
and drop you in a bash shell where you can run the datatracker.
|
||||
|
||||
6. You are now ready to run the tests::
|
||||
|
||||
(virtual) $ ietf/manage.py test --settings=settings_sqlitetest
|
||||
|
||||
and then start the dev server::
|
||||
|
||||
(virtual) $ ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
Note the IP address ``0.0.0.0`` used to make the dev server bind to all
|
||||
addresses. The internal port 8000 has been mapped to port 8000 externally,
|
||||
too. In order to find the IP address of the VirtualBox, run ``'$
|
||||
docker-machine ip'`` *outside* the virtual environment::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker-machine ip
|
||||
192.168.59.103
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ open http://192.168.59.103:8000/
|
||||
|
||||
.. _Docker: https://www.docker.com/
|
||||
.. _`List of Installation Instructions`: https://docs.docker.com/v1.8/installation/
|
||||
.. _VirtualBox: https://www.virtualbox.org/
|
||||
|
||||
|
||||
39
docker/build
39
docker/build
|
|
@ -1,11 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.11
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
export LANG=C
|
||||
|
|
@ -31,10 +31,10 @@ DESCRIPTION
|
|||
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
if [ "$(uname)" = "Linux" ] || [ "$(uname)" = "Darwin" ]; then
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
cat <<EOF
|
||||
|
||||
|
|
@ -48,7 +48,7 @@ COPYRIGHT
|
|||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
|
@ -59,10 +59,6 @@ function die() {
|
|||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
|
|
@ -77,24 +73,22 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
|
|||
|
||||
# Options
|
||||
shortopts=hlt:vV
|
||||
longopts=help,local,tag=,verbose,version
|
||||
longopts=help,local,tag=,version
|
||||
|
||||
# Default values
|
||||
BRANCH=$(svn log -v ^/tags -l 2 | grep 'A /tags/[1-9]' | awk '{print $2}')
|
||||
TAG=${BRANCH##*/}
|
||||
IMAGE=ietf/datatracker-environment
|
||||
TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}')")
|
||||
LOCAL=""
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
|
|
@ -102,7 +96,6 @@ while true ; do
|
|||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-l| --local) LOCAL=1;; # Don't upload
|
||||
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the latest svn tags name
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
|
|
@ -113,14 +106,10 @@ done
|
|||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
if [ "$(uname)" == "Linux" ]; then
|
||||
die "Didn't expect to run this script on Linux -- are you inside docker?"
|
||||
fi
|
||||
|
||||
docker rmi -f ietf/datatracker-environment:trunk || true
|
||||
docker build -t ietf/datatracker-environment:$TAG docker/
|
||||
docker tag $(docker images -q ietf/datatracker-environment | head -n 1) ietf/datatracker-environment:latest
|
||||
docker rmi -f $IMAGE:trunk 2>/dev/null || true
|
||||
docker build --progress plain -t "$IMAGE:$TAG" docker/
|
||||
docker tag "$(docker images -q $IMAGE | head -n 1)" $IMAGE:latest
|
||||
if [ -z "$LOCAL" ]; then
|
||||
docker push ietf/datatracker-environment:latest
|
||||
docker push ietf/datatracker-environment:$TAG
|
||||
docker push $IMAGE:latest
|
||||
docker push "$IMAGE:$TAG"
|
||||
fi
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$progdir/run ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
|
||||
|
|
@ -1,195 +1,85 @@
|
|||
#!/bin/bash
|
||||
|
||||
# A little bit of setup
|
||||
export LANG=en_GB.UTF-8
|
||||
MYSQLDIR=/var/lib/mysql
|
||||
|
||||
echo "Gathering info ..."
|
||||
MYSQLDIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
|
||||
if [ ! "$USER" ]; then
|
||||
echo "Environment variable USER is not set -- will set USER='django'."
|
||||
USER="django"
|
||||
fi
|
||||
if [ ! "$UID" ]; then
|
||||
echo "Environment variable UID is not set -- will set UID='1000'."
|
||||
UID="1000"
|
||||
fi
|
||||
if [ ! "$GID" ]; then
|
||||
echo "Environment variable GID is not set -- will set GID='1000'."
|
||||
GID="1000"
|
||||
fi
|
||||
if [ ! "$TAG" ]; then
|
||||
echo "Environment variable TAG is not set -- will set TAG='datatracker'."
|
||||
TAG="datatracker"
|
||||
fi
|
||||
echo "User $USER ($UID:$GID)"
|
||||
|
||||
echo "Checking if MySQL base data exists ..."
|
||||
if [ ! -d $MYSQLDIR/mysql ]; then
|
||||
echo "WARNING: Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
|
||||
if [ ! -d "$MYSQLDIR" ]; then
|
||||
echo "WARNING: Expected the directory $MYSQLDIR to exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting up the 'mysql' user for database file access ..."
|
||||
MYSQL_TARGET_GID=$(stat -c "%g" $MYSQLDIR/mysql)
|
||||
if ! grep -q ":$MYSQL_TARGET_GID:$" /etc/group; then
|
||||
groupadd -g $MYSQL_TARGET_GID mysqldata
|
||||
fi
|
||||
usermod -a -G $MYSQL_TARGET_GID mysql
|
||||
service rsyslog start
|
||||
|
||||
echo "Checking if MySQL is running ..."
|
||||
if ! /etc/init.d/mysql status; then
|
||||
echo "Starting mysql ..."
|
||||
/etc/init.d/mysql start
|
||||
if [ -z "$(ls -A $MYSQLDIR/mysql 2>/dev/null)" ]; then
|
||||
echo "WARNING: Database seems to be empty."
|
||||
mysql_install_db > /dev/null || exit 1
|
||||
fi
|
||||
|
||||
echo "Checking if syslogd is running ..."
|
||||
if ! /etc/init.d/rsyslog status > /dev/null; then
|
||||
echo "Starting syslogd ..."
|
||||
/etc/init.d/rsyslog start
|
||||
service mariadb start
|
||||
|
||||
if ! service mariadb status; then
|
||||
echo "ERROR: MySQL isn't running."
|
||||
grep mysqld /var/log/syslog
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Give debian-sys-maint access, to avoid complaints later
|
||||
mysql mysql <<< "GRANT ALL PRIVILEGES on *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '$(awk '/^password/ {print $3; exit}' /etc/mysql/debian.cnf )' WITH GRANT OPTION; FLUSH PRIVILEGES;"
|
||||
|
||||
echo "Checking if the IETF database exists at $MYSQLDIR ..."
|
||||
if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
|
||||
if [ -z "$DATADIR" ]; then
|
||||
echo "DATADIR is not set, but the IETF database needs to be set up -- can't continue, exiting the docker init script."
|
||||
exit 1
|
||||
fi
|
||||
ls -l $MYSQLDIR
|
||||
|
||||
if ! /etc/init.d/mysql status; then
|
||||
echo "Didn't find the IETF database, but can't set it up either, as MySQL isn't running."
|
||||
else
|
||||
echo "Creating database ..."
|
||||
mysqladmin -u root --default-character-set=utf8 create ietf_utf8
|
||||
|
||||
echo "Setting up permissions ..."
|
||||
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
|
||||
|
||||
echo "Fetching database ..."
|
||||
DUMPDIR=/home/$USER/$DATADIR
|
||||
wget -N -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz
|
||||
|
||||
echo "Loading database ..."
|
||||
gunzip < $DUMPDIR/ietf_utf8.sql.gz \
|
||||
| pv --progress --bytes --rate --eta --cursor --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| mysql --user=django --password=RkTkDPFnKpko -s -f ietf_utf8 \
|
||||
&& rm /tmp/ietf_utf8.sql.gz
|
||||
fi
|
||||
echo "WARNING: IETF database seems to be missing; populating it from dump."
|
||||
mysqladmin -u root --default-character-set=utf8 create ietf_utf8
|
||||
pushd /mariadb-sys-master || exit
|
||||
mysql -u root < sys_10.sql
|
||||
popd || exit
|
||||
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON *.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
|
||||
/root/src/docker/updatedb
|
||||
fi
|
||||
|
||||
if ! grep -q ":$GID:$" /etc/group ; then
|
||||
echo "Creating group entry for GID '$GID' ..."
|
||||
groupadd -g "$GID" "$USER"
|
||||
fi
|
||||
if ! id -u "$USER" &> /dev/null; then
|
||||
echo "Creating user '$USER' ..."
|
||||
useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER
|
||||
echo "$USER:$USER" | chpasswd
|
||||
fi
|
||||
|
||||
VIRTDIR="/opt/home/$USER/$TAG"
|
||||
echo "Checking that there's a virtual environment for $TAG ..."
|
||||
if [ ! -f $VIRTDIR/bin/activate ]; then
|
||||
echo "Setting up python virtualenv at $VIRTDIR ..."
|
||||
mkdir -p $VIRTDIR
|
||||
python3.6 -m venv $VIRTDIR
|
||||
echo -e "
|
||||
# This is from $VIRTDIR/bin/activate, to activate the
|
||||
# datatracker virtual python environment on docker container entry:
|
||||
" >> /etc/bash.bashrc
|
||||
cat $VIRTDIR/bin/activate >> /etc/bash.bashrc
|
||||
cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc
|
||||
else
|
||||
echo "Using virtual environment at $VIRTDIR"
|
||||
fi
|
||||
|
||||
echo "Activating the virtual python environment ..."
|
||||
. $VIRTDIR/bin/activate
|
||||
|
||||
|
||||
if ! $VIRTDIR/bin/python -c "import django"; then
|
||||
echo "Installing requirements ..."
|
||||
pip install --upgrade pip
|
||||
reqs=/home/$USER/$CWD/requirements.txt
|
||||
if [ ! -f $reqs ]; then
|
||||
echo " Using $reqs"
|
||||
pip install -r $reqs
|
||||
else
|
||||
echo " Didn't find $reqs"
|
||||
echo " Using /usr/local/share/datatracker/requirements.txt"
|
||||
pip install -r /usr/local/share/datatracker/requirements.txt
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f /home/$USER/$CWD/ietf/settings_local.py ]; then
|
||||
echo "Setting up a default settings_local.py ..."
|
||||
cp /home/$USER/$CWD/docker/settings_local.py /home/$USER/$CWD/ietf/settings_local.py
|
||||
fi
|
||||
|
||||
for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do
|
||||
dir="/home/$USER/$CWD/$sub"
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Creating dir $dir"
|
||||
mkdir -p "$dir";
|
||||
fi
|
||||
done
|
||||
|
||||
for sub in \
|
||||
nomcom_keys/public_keys \
|
||||
developers/ietf-ftp \
|
||||
developers/ietf-ftp/bofreq \
|
||||
developers/ietf-ftp/charter \
|
||||
developers/ietf-ftp/conflict-reviews \
|
||||
developers/ietf-ftp/internet-drafts \
|
||||
developers/ietf-ftp/rfc \
|
||||
developers/ietf-ftp/status-changes \
|
||||
developers/ietf-ftp/yang/catalogmod \
|
||||
developers/ietf-ftp/yang/draftmod \
|
||||
developers/ietf-ftp/yang/ianamod \
|
||||
developers/ietf-ftp/yang/invalmod \
|
||||
developers/ietf-ftp/yang/rfcmod \
|
||||
developers/www6s \
|
||||
developers/www6s/staging \
|
||||
developers/www6s/wg-descriptions \
|
||||
developers/www6s/proceedings \
|
||||
developers/www6/ \
|
||||
developers/www6/iesg \
|
||||
developers/www6/iesg/evaluation \
|
||||
test/id \
|
||||
test/staging \
|
||||
test/archive \
|
||||
test/rfc \
|
||||
test/media \
|
||||
test/wiki/ietf \
|
||||
data/nomcom_keys/public_keys \
|
||||
data/developers/ietf-ftp \
|
||||
data/developers/ietf-ftp/bofreq \
|
||||
data/developers/ietf-ftp/charter \
|
||||
data/developers/ietf-ftp/conflict-reviews \
|
||||
data/developers/ietf-ftp/internet-drafts \
|
||||
data/developers/ietf-ftp/rfc \
|
||||
data/developers/ietf-ftp/status-changes \
|
||||
data/developers/ietf-ftp/yang/catalogmod \
|
||||
data/developers/ietf-ftp/yang/draftmod \
|
||||
data/developers/ietf-ftp/yang/ianamod \
|
||||
data/developers/ietf-ftp/yang/invalmod \
|
||||
data/developers/ietf-ftp/yang/rfcmod \
|
||||
data/developers/www6s \
|
||||
data/developers/www6s/staging \
|
||||
data/developers/www6s/wg-descriptions \
|
||||
data/developers/www6s/proceedings \
|
||||
data/developers/www6/ \
|
||||
data/developers/www6/iesg \
|
||||
data/developers/www6/iesg/evaluation \
|
||||
; do
|
||||
dir="/home/$USER/$CWD/data/$sub"
|
||||
dir="/root/src/$sub"
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Creating dir $dir"
|
||||
echo "Creating dir $dir"
|
||||
mkdir -p "$dir";
|
||||
chown "$USER" "$dir"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ! -f "/home/$USER/$CWD/test/data/draft-aliases" ]; then
|
||||
echo "Generating draft aliases ..."
|
||||
ietf/bin/generate-draft-aliases }
|
||||
if [ ! -f /root/src/ietf/settings_local.py ]; then
|
||||
echo "Setting up a default settings_local.py ..."
|
||||
cp /root/src/docker/settings_local.py /root/src/ietf/settings_local.py
|
||||
fi
|
||||
|
||||
if [ ! -f "/home/$USER/$CWD/test/data/group-aliases" ]; then
|
||||
echo "Generating group aliases ..."
|
||||
ietf/bin/generate-wg-aliases }
|
||||
fi
|
||||
python -m smtpd -n -c DebuggingServer localhost:2025 &
|
||||
echo
|
||||
|
||||
chown -R $USER /opt/home/$USER
|
||||
chmod -R g+w /usr/local/lib/ # so we can patch libs if needed
|
||||
|
||||
cd "/home/$USER/$CWD" || cd "/home/$USER/"
|
||||
|
||||
echo "Done!"
|
||||
if ! echo "$LANG" | grep "UTF-8"; then
|
||||
echo ""
|
||||
echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc"
|
||||
if [ -z "$*" ]; then
|
||||
bash
|
||||
else
|
||||
echo "LANG=$LANG"
|
||||
bash -c "$*"
|
||||
fi
|
||||
|
||||
HOME=/opt/home/$USER
|
||||
|
||||
su -p $USER
|
||||
service mariadb stop
|
||||
service rsyslog stop
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
sudo su - -c "apt-get update \
|
||||
&& apt-get install -qy \
|
||||
apt-file \
|
||||
apt-show-versions \
|
||||
graphviz \
|
||||
ghostscript \
|
||||
apache2-utils \
|
||||
chromium-driver \
|
||||
enscript \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*"
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$progdir/run python -m smtpd -n -c DebuggingServer localhost:2025
|
||||
205
docker/run
205
docker/run
|
|
@ -1,13 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.20
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo $parent | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
|
|
@ -30,10 +30,10 @@ DESCRIPTION
|
|||
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
if [ "$(uname)" = "Linux" ] || [ "$(uname)" = "Darwin" ]; then
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
cat <<EOF
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ COPYRIGHT
|
|||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
|
@ -58,10 +58,6 @@ function die() {
|
|||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
|
|
@ -75,48 +71,32 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
|
|||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=dhi:m:Mp:r:t:vVu:cC
|
||||
longopts=download-data,help,ietfdb-url=,mysqldata=,no-mysqldir,port=,docker-repo=,tag=,verbose,version,user=,cached,no-cached
|
||||
shortopts=hp:vVcC
|
||||
longopts=help,port=,version,cached,no-cached
|
||||
|
||||
# Default values
|
||||
MYSQLDIR=$parent/data/mysql
|
||||
FILEDIR=$parent/../data/
|
||||
PORT=8000
|
||||
REPO="ietf/datatracker-environment"
|
||||
DBURL=https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
|
||||
WHO=$(whoami)
|
||||
CACHED=''
|
||||
CACHED=':cached'
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
if [ "$(uname)" = "Darwin" ]; then
|
||||
CACHED=':cached'
|
||||
fi
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-c| --cached) CACHED=':cached';; # Use cached disk access to reduce system load
|
||||
-C| --no-cached) CACHED='';; # Use fully synchronized disk access
|
||||
-d| --download-data) DOWNLOAD=1;; # Download and set up the database files
|
||||
-C| --no-cached) CACHED=':consistent';; # Use fully synchronized disk access
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-f| --filedir) FILEDIR=$2; shift;; # Set the desired location of drafts, charters etc.
|
||||
-i| --ietfdb-url) DBURL=$2; shift;; # Use an alternative database tarball URL
|
||||
-m| --mysqldir) MYSQLDIR=$2; shift;; # Set the desired location of MySQL's database files
|
||||
-p| --port) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
|
||||
-r| --docker-repo) REPO=$2; shift;; # Use the given docker repository, instead of the default
|
||||
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the svn branch name
|
||||
-u| --user) WHO=$2; shift;; # Run the container as the specified user
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
|
|
@ -125,163 +105,14 @@ while true ; do
|
|||
done
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
TAG=$(basename $(svn info $parent | grep ^URL | awk '{print $2}'))
|
||||
TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}')")
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
[ -f /proc/1/cgroups ] && grep 'docker' /proc/1/cgroups && die "It looks like you're inside docker already ..."
|
||||
|
||||
if [ "$(uname)" = "Darwin" ]; then
|
||||
APP="/Applications/Docker.app"
|
||||
CMD="open -a"
|
||||
elif [ "$(uname)" = "Linux" ]; then
|
||||
echo "Running on Linux."
|
||||
elif [[ $(uname) =~ CYGWIN.* ]]; then
|
||||
echo "Running under Cygwin."
|
||||
CMD="echo"
|
||||
MYSQLDIR=$(echo $MYSQLDIR | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
|
||||
WHO=$(echo $WHO | sed -e 's/^.*\\//' | tr -d \\r)
|
||||
DRIVE=$(echo $USERPROFILE | sed -e 's/\(.\).*/\1/' -e 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefdhijklmnopqartuvwxyz/')
|
||||
HOME=$DRIVE$(echo $USERPROFILE | sed -e 's/\\/\//g' -e's/.\(.*\)/\1/')
|
||||
echo "Using home dir $HOME"
|
||||
PWD=$(echo $PWD | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
|
||||
if [ "${PWD#$HOME}" = "$PWD" ]; then
|
||||
die "You must work inside your home directory ($HOME)"
|
||||
fi
|
||||
CGWPARENT=$(echo $parent | sed -e 's/^\(.\)\:/\/cygdrive\/\1/')
|
||||
ICSFILES=$(/usr/bin/find $CGWPARENT/vzic/zoneinfo/ -name '*.ics' -print)
|
||||
for ICSFILE in $ICSFILES
|
||||
do
|
||||
LINK=$(head -n1 $ICSFILE | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
|
||||
if [ "$LINK" ]; then
|
||||
WDIR=$(dirname $ICSFILE)
|
||||
echo "Replacing $(basename $ICSFILE) with $LINK"
|
||||
cp -f $WDIR/$LINK $ICSFILE
|
||||
fi
|
||||
done
|
||||
else
|
||||
die "This script does not have support for your architecture ($(uname)); sorry :-("
|
||||
fi
|
||||
|
||||
WHOUID=$(id -u $WHO)
|
||||
WHOGID=$(id -g $WHO)
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
echo "Not trying to start a virtual docker machine on Linux"
|
||||
elif [[ $(uname) =~ CYGWIN.* ]]; then
|
||||
if ! docker info 1> /dev/null 2>&1; then
|
||||
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
|
||||
die "Don't know how to start docker when running under Cygwin"
|
||||
fi
|
||||
TAG=$(echo $TAG | tr -d \\r)
|
||||
URL="http://localhost:$PORT/"
|
||||
elif [ -e "$APP" ]; then
|
||||
info=$(docker info 2>&1 || true)
|
||||
if ! docker info 1> /dev/null 2>&1; then
|
||||
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
|
||||
echo "Will attempt to start docker by doing '\$ $CMD $APP':"
|
||||
$CMD $APP
|
||||
declare -i count
|
||||
printf "Waiting for docker engine .."
|
||||
while true; do
|
||||
printf "."
|
||||
sleep 2
|
||||
if docker info >/dev/null 2>&1; then break; fi
|
||||
count=$(( $count + 1))
|
||||
if [ $count -gt 10 ]; then
|
||||
die "No contact with docker engine; bailing out."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
URL="http://localhost:$PORT/"
|
||||
else
|
||||
if [ $($machine status default | tr "A-Z" "a-z" ) != "running" ]; then
|
||||
echo "The docker VM doesn't seem to be running; will attempt to start it by doing '\$ $machine start':"
|
||||
$machine start || die "Failed taking up the Docker VM"
|
||||
fi
|
||||
|
||||
if [ -f "$machine" ]; then
|
||||
if [ $($machine status default | tr "A-Z" "a-z") = "running" ]; then
|
||||
tmpfile=$(mktemp docker.run.XXXXXXXX)
|
||||
if $machine env 2>/dev/null | grep DOCKER_ > $tmpfile; then
|
||||
mv $tmpfile ~/.docker-info
|
||||
elif printenv | grep DOCKER_ > $tmpfile; then
|
||||
mv $tmpfile ~/.docker-info
|
||||
else
|
||||
rm $tmpfile
|
||||
die "Failed setting the appropriate DOCKER_* environment variables."
|
||||
fi
|
||||
. ~/.docker-info
|
||||
else
|
||||
rm -f ~/.docker-info
|
||||
fi
|
||||
ip=$($machine ip)
|
||||
URL="http://$ip:$PORT/"
|
||||
fi
|
||||
fi
|
||||
|
||||
image=$(docker ps | grep "$REPO:$TAG" | awk '{ print $1 }')
|
||||
if [ "$image" ]; then
|
||||
if [ "$*" ]; then
|
||||
echo "Running 'cd ~/${parent#$HOME/}; $*'"
|
||||
docker exec -u "$WHO" -t $image bash -i -c "cd; cd \"${parent#$HOME/}\"; $*"
|
||||
else
|
||||
docker exec -u "$WHO" -ti $image bash -i
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo "Starting a docker container for '$TAG'."
|
||||
|
||||
if [ -n "$DOWNLOAD" ]; then
|
||||
(
|
||||
cd "$(dirname $MYSQLDIR)"
|
||||
wget -N "$DBURL" && tar xjf ietf_utf8.bin.tar.bz2 && chmod -R go+rwX mysql
|
||||
)
|
||||
[ -d "$MYSQLDIR" ] || die "The download seems to have failed; still no $MYSQLDIR. Giving up."
|
||||
else
|
||||
[ -d "$MYSQLDIR" ] || die "Expected $MYSQLDIR to exist, but it\ndidn't. Use '$program -d' to download and unpack the database."
|
||||
fi
|
||||
|
||||
if ! docker images "$REPO" | grep -q "$TAG"; then
|
||||
echo "Fetching docker image '$REPO:$TAG'"
|
||||
if ! docker pull "$REPO:$TAG"; then
|
||||
docker pull "$REPO:latest" || die "Failed to pull down the '$REPO:latest' docker image"
|
||||
id=$(docker images "$REPO" | grep latest | awk '{print $3}')
|
||||
echo "Tagging $REPO:latest as $REPO:$TAG for use as environment for this branch."
|
||||
docker tag $id "$REPO:$TAG"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "\nThe web interface for 'runserver' should appear on $URL\n"
|
||||
echo -e "User $WHO ($WHOUID:$WHOGID)"
|
||||
if [ -z "$MYSQLDIR" ]; then
|
||||
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
|
||||
-e USER="$WHO" -e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" \
|
||||
-e TAG="$TAG" -e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
|
||||
"$REPO:$TAG" "$@"
|
||||
else
|
||||
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
|
||||
-v "$MYSQLDIR:/var/lib/mysql" -e USER="$WHO" \
|
||||
-e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" -e TAG="$TAG" \
|
||||
-e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
|
||||
"$REPO:$TAG" "$@"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Committing changes in the container to an image:"
|
||||
image=$( docker images -q $REPO:$TAG)
|
||||
latest=$(docker ps -lq -f "ancestor=$image")
|
||||
docker commit $latest $REPO:$TAG
|
||||
|
||||
echo ""
|
||||
echo "Cleaning up containers and images"
|
||||
docker rm $latest
|
||||
DANGLING=$( docker images -f dangling=true -q )
|
||||
if [ -n "$DANGLING" ]; then
|
||||
echo "Dangling images: $DANGLING"
|
||||
docker rmi -f $DANGLING
|
||||
fi
|
||||
|
||||
fi
|
||||
echo "Starting a docker container for '$REPO:$TAG'."
|
||||
mkdir -p "$MYSQLDIR"
|
||||
docker run -ti -p "$PORT":8000 -p 33306:3306 \
|
||||
-v "$parent:/root/src$CACHED" \
|
||||
-v "$MYSQLDIR:/var/lib/mysql:delegated" \
|
||||
"$REPO:$TAG" "$@"
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
|
||||
# set a fancy prompt
|
||||
PS_TIME="\A "
|
||||
|
||||
if [ "$(whoami)" = "root" ]; then
|
||||
PS_COLOUR='\[\033[1;31m\]'
|
||||
PS_END="\[\033[m\]"
|
||||
else
|
||||
PS_COLOUR="\[\033[1;34m\]"
|
||||
PS_END="\[\033[m\]"
|
||||
fi
|
||||
|
||||
if [ "$TERM" = xterm ]; then
|
||||
PS_XTERM="\[\033]0;\]\h:\w\007"
|
||||
else
|
||||
PS_XTERM=""
|
||||
fi
|
||||
COLOUR_RED='\[\033[1;31m\]'
|
||||
COLOUR_BLK='\[\033[1;30m\]'
|
||||
if [ "$SHLVL" -gt 1 ]; then
|
||||
PS_SHLVL="$(eval "printf '>%.0s' {2..$SHLVL}") "
|
||||
else
|
||||
PS_SHLVL=""
|
||||
fi
|
||||
|
||||
PS1="$PS_XTERM\n$COLOUR_RED$PS_SHLVL$PS_COLOUR$PS_TIME$COLOUR_RED$PS_COLOUR${VIRTUAL_ENV:+$COLOUR_RED($(basename $VIRTUAL_ENV))$PS_COLOUR }\w\n\u @ $COLOUR_BLK\h$PS_COLOUR \\$ $PS_END"
|
||||
114
docker/setupdb
114
docker/setupdb
|
|
@ -1,114 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$program - Set up a local copy of the IETF database MySQL files
|
||||
|
||||
SYNOPSIS
|
||||
$program [OPTIONS] ARGS
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script downloads a prebuilt copy of the IETF database MySQL
|
||||
files, ready for mapping into the /var/lib/mysql/ directory of the
|
||||
ietf/database-environment Docker image.
|
||||
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=hvV
|
||||
longopts=help,verbose,version
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
[ -n "$MYSQLDIR" ] || MYSQLDIR=$parent/data/mysql
|
||||
[ -n "$URL" ] || URL=rsync.ietf.org::dev.db/ietf_utf8.bin.tar.bz2
|
||||
|
||||
cd $(dirname $MYSQLDIR)
|
||||
echo "Downloading a MySQL database image ..."
|
||||
set $(rsync --version | head -n1 | awk '{print $3}' | tr '.' ' ')
|
||||
maj=$1; min=$2; fix=$3
|
||||
if [ "$maj" -ge "3" -a "$min" -ge "1" ]; then
|
||||
rsync --info=progress2 $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
|
||||
else
|
||||
rsync -v $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
|
||||
fi
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
#!/bin/bash
|
||||
# -*- indent-with-tabs: 1 -*-
|
||||
|
||||
version=0.10
|
||||
version=0.20
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
|
@ -26,9 +26,9 @@ DESCRIPTION
|
|||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
grep -E "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" "$0" | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
cat <<EOF
|
||||
|
||||
|
|
@ -42,21 +42,17 @@ COPYRIGHT
|
|||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
|
|
@ -71,24 +67,21 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
|
|||
|
||||
# Options
|
||||
shortopts=DLZhqvV
|
||||
longopts=no-download,no-load,no-zap,help,quiet,verbose,version
|
||||
longopts=no-download,no-load,no-zap,help,version
|
||||
|
||||
LOAD=1
|
||||
DOWNLOAD=1
|
||||
DROP=1
|
||||
QUIET=""
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n "$program" -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
|
|
@ -97,8 +90,6 @@ while true ; do
|
|||
-L| --no-load) LOAD=""; ;; # Don't load the database
|
||||
-Z| --no-zap) DROP="";; # Don't drop new tables
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-q| --quiet) QUIET="-q";; # Don't show progress
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
|
|
@ -109,50 +100,27 @@ done
|
|||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
echo "Gathering info ..."
|
||||
MYSQLDIR="$(/usr/sbin/mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
|
||||
MYSQLDIR=${MYSQLDIR%/}
|
||||
DATADIR=$parent/data
|
||||
|
||||
# echo "Checking if MySQL base data exists ..."
|
||||
# if [ ! -d $MYSQLDIR/mysql ]; then
|
||||
# die "Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
|
||||
# fi
|
||||
|
||||
|
||||
# echo "Checking if the IETF database exists at $MYSQLDIR ..."
|
||||
# if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
|
||||
# echo "Creating database ..."
|
||||
# mysqladmin -u root --default-character-set=utf8 create ietf_utf8
|
||||
#
|
||||
# echo "Setting up permissions ..."
|
||||
# mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
|
||||
# fi
|
||||
|
||||
DUMP=ietf_utf8.sql.gz
|
||||
if [ "$DOWNLOAD" ]; then
|
||||
echo "Fetching database dump ..."
|
||||
wget $QUIET -N -P $DATADIR https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz || die "No new data, quitting."
|
||||
echo "Fetching database dump..."
|
||||
rsync --info=progress2 rsync.ietf.org::dev.db/$DUMP "$DATADIR"
|
||||
fi
|
||||
|
||||
if [ "$LOAD" ]; then
|
||||
echo "Loading database ..."
|
||||
if [ -z "$QUIET" ]; then
|
||||
gunzip < $DATADIR/ietf_utf8.sql.gz \
|
||||
| pv --progress --bytes --rate --eta --size $(gzip --list --quiet $DATADIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| $parent/ietf/manage.py dbshell
|
||||
else
|
||||
gunzip < $DATADIR/ietf_utf8.sql.gz \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| $parent/ietf/manage.py dbshell
|
||||
fi
|
||||
|
||||
echo "Loading database..."
|
||||
SIZE=$(pigz --list "$DATADIR/$DUMP" | tail -n 1 | awk '{ print $2 }')
|
||||
pigz -d < "$DATADIR/$DUMP" \
|
||||
| pv --progress --bytes --rate --eta --size "$SIZE" \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| "$parent/ietf/manage.py" dbshell
|
||||
fi
|
||||
|
||||
if [ "$DROP" ]; then
|
||||
echo "Dropping tables not in the dump (so migrations can succeed) ..."
|
||||
diff <(zcat $DATADIR/ietf_utf8.sql.gz | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
|
||||
<($parent/ietf/manage.py dbshell <<< 'show tables;' | tail -n +2) \
|
||||
echo "Dropping tables not in the dump (so migrations can succeed)..."
|
||||
diff <(pigz -d -c "$DATADIR/$DUMP" | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
|
||||
<("$parent/ietf/manage.py" dbshell <<< 'show tables;' | tail -n +2) \
|
||||
| grep '^>' | awk '{print "drop table if exists", $2, ";";}' \
|
||||
| tee >(cat >&2) | $parent/ietf/manage.py dbshell
|
||||
| tee >(cat >&2) | "$parent/ietf/manage.py" dbshell
|
||||
fi
|
||||
Loading…
Reference in a new issue