Merged branch ^/personal/lars/7.39.1.dev0@19465 from lars@eggert.org:
Rework the docker container. - Legacy-Id: 19507
This commit is contained in:
Robert Sparks
commit
2b478da42e
|
|
@ -1,185 +1,132 @@
|
|||
# -*- shell-mode -*-
|
||||
# This file is a docker (https://www.docker.com/what-docker) recipe, which can be used to build
|
||||
# a docker image which is ready to run a datatracker in development mode.
|
||||
# This is a Dockerfile with everything in it to run the IETF datatracker.
|
||||
#
|
||||
# It is used to build an image (once you've installed docker) using a command like this (assuming
|
||||
# suitable replacement of $variables:
|
||||
# If you make changes to the datatracker that add new dependencies (python
|
||||
# packages or otherwise), you need to rebuild this image to make them
|
||||
# available. Do this in the top-level directory of your datatracker source
|
||||
# tree:
|
||||
#
|
||||
# $ docker build -t $yourdockerid/datatracker:$version
|
||||
# docker/build
|
||||
#
|
||||
# To use a pre-built image, assuming we're on OS X and have a checked-out datatracker repository
|
||||
# at /Users/$login/src/6.8.1.dev0, you would start (again assuming you've installed docker)
|
||||
# a container from an image, as follows:
|
||||
#
|
||||
# $ docker run -ti --name=$containername -v /Users/$login:/home/$login levkowetz/datatracker:$version /bin/bash
|
||||
#
|
||||
# This maps your home directory to /home/$login in the container, and starts it running /bin/bash.
|
||||
#
|
||||
# In this first version, the docker environment is set up so that tests will run successfully,
|
||||
# but the database has *not* been loaded with a dump, and supporting files (drafts, charters, etc.)
|
||||
# have *not* been downloaded.
|
||||
# You can then execute the datatracker like this (also from the top-level
|
||||
# datatracker source directory):
|
||||
#
|
||||
# docker/run
|
||||
|
||||
FROM dyne/devuan:beowulf
|
||||
LABEL maintainer="Henrik Levkowetz <henrik@levkowetz.com>"
|
||||
FROM ubuntu:hirsute
|
||||
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
|
||||
|
||||
# Default django runserver port
|
||||
EXPOSE 8000
|
||||
EXPOSE 8000
|
||||
|
||||
# Default mysqld/mariadb port
|
||||
EXPOSE 3306
|
||||
|
||||
# Run apt-get noninteractive
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV DEVUAN_FRONTEND=noninteractive
|
||||
|
||||
# Uncomment this to be able to install and run apt-show-versions:
|
||||
RUN rm -v /etc/apt/apt.conf.d/docker-compress
|
||||
RUN rm -v /var/lib/apt/lists/*lz4
|
||||
|
||||
RUN apt-get update --allow-releaseinfo-change
|
||||
RUN apt-get install -qy apt-transport-https
|
||||
|
||||
# Use backports, updates, and security updates
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf main contrib non-free" > /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-security main contrib non-free" >> /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-updates main contrib non-free" >> /etc/apt/sources.list
|
||||
RUN echo "deb http://deb.devuan.org/merged beowulf-backports main contrib non-free" >> /etc/apt/sources.list
|
||||
|
||||
# Remove some excludes for the docker image
|
||||
RUN sed -i -e '/^path-exclude=.*\/groff/d' \
|
||||
-e '/^path-exclude=.*\/locale/d' \
|
||||
-e '/^path-exclude=.*\/man/d' /etc/dpkg/dpkg.cfg.d/docker-excludes
|
||||
|
||||
# Install needed packages
|
||||
#
|
||||
# We're not including graphviz and ghostscript, needed for the 3 document
|
||||
# dependency graph tests; they would increase the size of the image by about
|
||||
# 15%, about 100MB.
|
||||
|
||||
# Fetch apt package information, and upgrade to latest package versions
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get -qy upgrade
|
||||
|
||||
# Install the packages we need
|
||||
RUN apt-get install -qy \
|
||||
build-essential \
|
||||
bzip2 \
|
||||
ca-certificates \
|
||||
colordiff \
|
||||
gawk \
|
||||
gcc \
|
||||
ipython \
|
||||
jq \
|
||||
less \
|
||||
libbz2-dev \
|
||||
libdb5.3-dev \
|
||||
libexpat1-dev \
|
||||
libffi-dev \
|
||||
libgdbm-dev \
|
||||
libjpeg62-turbo-dev \
|
||||
liblzma-dev \
|
||||
libmagic1 \
|
||||
libmariadbclient-dev \
|
||||
libncurses5-dev \
|
||||
libncursesw5-dev \
|
||||
libreadline-dev \
|
||||
libsqlite3-dev \
|
||||
libssl-dev \
|
||||
libsvn1 \
|
||||
libxml2-dev \
|
||||
libxslt-dev \
|
||||
libz-dev \
|
||||
libffi-dev \
|
||||
locales \
|
||||
make \
|
||||
man \
|
||||
mariadb-client \
|
||||
mariadb-server \
|
||||
openssh-client \
|
||||
patch \
|
||||
procps \
|
||||
pv \
|
||||
rsync \
|
||||
RUN apt-get -y update && \
|
||||
# apt-get upgrade is normally not a good idea, but this is a dev container
|
||||
apt-get upgrade && \
|
||||
# Install all dependencies that are available as packages
|
||||
apt-get -y install --no-install-recommends \
|
||||
apache2-utils \
|
||||
apt-file \
|
||||
apt-utils \
|
||||
curl \
|
||||
enscript \
|
||||
gcc \
|
||||
ghostscript \
|
||||
git \
|
||||
gnupg \
|
||||
graphviz \
|
||||
libmagic-dev \
|
||||
libmariadb-dev \
|
||||
locales \
|
||||
mariadb-server \
|
||||
npm \
|
||||
pigz \
|
||||
pv \
|
||||
python-is-python3 \
|
||||
python3-dev \
|
||||
python3-pip \
|
||||
rsyslog \
|
||||
subversion \
|
||||
sudo \
|
||||
uuid-dev \
|
||||
vim \
|
||||
wget \
|
||||
xz-utils\
|
||||
zile \
|
||||
zlib1g-dev
|
||||
unzip \
|
||||
yang-tools && \
|
||||
# Since snap doesn't work in Docker containers, install chromedriver per
|
||||
# https://gist.github.com/varyonic/dea40abcf3dd891d204ef235c6e8dd79#gistcomment-3160722
|
||||
curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \
|
||||
echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y --no-install-recommends google-chrome-stable && \
|
||||
CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") && \
|
||||
DRIVERVER=$(curl "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") && \
|
||||
curl -L -O -C - "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" && \
|
||||
unzip chromedriver_linux64.zip -d /bin && \
|
||||
rm chromedriver_linux64.zip && \
|
||||
# Install some other packages that are not dependencies but make life easier
|
||||
apt-get -y install --no-install-recommends \
|
||||
fish \
|
||||
less \
|
||||
nano \
|
||||
ripgrep \
|
||||
zsh && \
|
||||
# Reduce image footprint (not that it matters given the size of the above)
|
||||
apt-get -y clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Postgresql packages
|
||||
RUN apt-get install -qy \
|
||||
postgresql-11 \
|
||||
postgresql-server-dev-11
|
||||
# Set locale to en_US.UTF-8
|
||||
RUN dpkg-reconfigure locales && \
|
||||
locale-gen en_US.UTF-8 && \
|
||||
update-locale LC_ALL en_US.UTF-8
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
# Get the key used to sign the libyang repo
|
||||
RUN wget -nv http://download.opensuse.org/repositories/home:liberouter/Debian_9.0/Release.key
|
||||
RUN apt-key add - < Release.key
|
||||
RUN rm Release.key
|
||||
# Install bower
|
||||
RUN npm install -g bower
|
||||
|
||||
# Add apt source entry for libyang
|
||||
RUN echo "deb http://download.opensuse.org/repositories/home:/liberouter/Debian_9.0/ /" >> /etc/apt/sources.list.d/libyang.list
|
||||
|
||||
# Update the package defs, and install the desired mysql from the mysql repo
|
||||
RUN apt-get update
|
||||
RUN apt-get install -qy libyang1
|
||||
|
||||
# This is expected to exist by the mysql startup scripts:
|
||||
#RUN touch /etc/mysql/debian.cnf
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Get rid of installation files we don't need in the image, to reduce size
|
||||
RUN apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Enable some common locales
|
||||
RUN sed -i -e 's/^. en_US/en_US/' -e 's/^. en_GB/en_GB/' -e 's/^. en_IE/en_IE/' /etc/locale.gen
|
||||
RUN locale-gen
|
||||
|
||||
# Remove an rsyslog module that we don't need, which also requires extra permissions
|
||||
RUN sed -i -e '/load="imklog"/d' /etc/rsyslog.conf
|
||||
|
||||
# Set up root password
|
||||
RUN echo "root:root" | chpasswd
|
||||
|
||||
# MySQL
|
||||
VOLUME /var/lib/mysql
|
||||
|
||||
# idnits and dependencies
|
||||
# Install idnits
|
||||
ADD https://tools.ietf.org/tools/idnits/idnits /usr/local/bin/
|
||||
RUN chmod +rx /usr/local/bin/idnits
|
||||
|
||||
# Directory for Mac certs
|
||||
RUN mkdir /etc/certificates
|
||||
# Install current datatracker python dependencies
|
||||
COPY requirements.txt /
|
||||
RUN pip install -r /requirements.txt
|
||||
|
||||
# # Python 3
|
||||
# # Comment in if OS does not provide python3.6, which is the current
|
||||
# # production version
|
||||
ENV PYVER=3.6.10
|
||||
ENV PYREV=3.6
|
||||
# Turn off rsyslog kernel logging (doesn't work in Docker)
|
||||
RUN sed -i '/imklog/s/^/#/' /etc/rsyslog.conf
|
||||
|
||||
WORKDIR /usr/src
|
||||
RUN wget -q https://www.python.org/ftp/python/$PYVER/Python-$PYVER.tar.xz
|
||||
RUN tar xJf Python-$PYVER.tar.xz
|
||||
RUN rm Python-$PYVER.tar.xz
|
||||
WORKDIR /usr/src/Python-$PYVER/
|
||||
RUN ./configure
|
||||
RUN make
|
||||
RUN make altinstall
|
||||
WORKDIR /usr/src
|
||||
RUN rm -rf /usr/src/Python-$PYVER/
|
||||
# Allow access to mariadb over the network
|
||||
RUN sed -i 's/127.0.0.1/0.0.0.0/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
|
||||
ENV HOSTNAME="datatracker"
|
||||
# Turn on mariadb performance_schema
|
||||
RUN sed -i 's/\[mysqld\]/\[mysqld\]\nperformance_schema=ON/' /etc/mysql/mariadb.conf.d/50-server.cnf
|
||||
|
||||
ENV DDIR="/usr/local/share/datatracker"
|
||||
RUN mkdir -p $DDIR
|
||||
WORKDIR $DDIR
|
||||
# Make the mariadb sys schema available for possible installation
|
||||
# We would normally use the next line, but that has a bug:
|
||||
# ADD https://github.com/FromDual/mariadb-sys/archive/master.zip /
|
||||
# This is the repo that has the PR:
|
||||
ADD https://github.com/grooverdan/mariadb-sys/archive/refs/heads/master.zip /
|
||||
RUN unzip /master.zip
|
||||
RUN rm /master.zip
|
||||
|
||||
COPY requirements.txt ./
|
||||
COPY setprompt ./
|
||||
# Colorize the bash shell
|
||||
RUN sed -i 's/#force_color_prompt=/force_color_prompt=/' /root/.bashrc
|
||||
|
||||
# Make a database dump available as part of the image, for if a user doesn't
|
||||
# have one installed locally yet - this saves a bunch of time then
|
||||
ADD https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz /
|
||||
RUN pigz -v -d /ietf_utf8.sql.gz && \
|
||||
sed -i -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' /ietf_utf8.sql
|
||||
RUN service mariadb start && \
|
||||
echo "This sequence will take a long time, please be patient" && \
|
||||
mysqladmin -u root --default-character-set=utf8 create ietf_utf8 && \
|
||||
bash -c "cd /mariadb-sys-master && mysql --user root < sys_10.sql" && \
|
||||
bash -c "mysql --user root ietf_utf8 <<< \"GRANT ALL PRIVILEGES ON *.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;\"" && \
|
||||
bash -c "mysql --user=django --password=RkTkDPFnKpko -f ietf_utf8 < /ietf_utf8.sql" && \
|
||||
service mariadb stop && \
|
||||
rm -rf /ietf_utf8.sql /mariadb-sys-master && \
|
||||
mv /var/lib/mysql /
|
||||
|
||||
# Copy the startup file
|
||||
COPY docker-init.sh /docker-init.sh
|
||||
RUN chmod +x /docker-init.sh
|
||||
ENTRYPOINT ["/docker-init.sh"]
|
||||
|
||||
CMD /bin/bash
|
||||
WORKDIR /root/src
|
||||
ENTRYPOINT ["/docker-init.sh"]
|
||||
56
docker/README.md
Normal file
56
docker/README.md
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
# Datatracker Development in Docker
|
||||
|
||||
## Getting started
|
||||
|
||||
1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred
|
||||
platform.
|
||||
|
||||
2. If you have a copy of the datatracker code checked out already, simply `cd`
|
||||
to the top-level directory.
|
||||
|
||||
If not, check out a datatracker branch as usual. We'll check out `trunk`
|
||||
below, but you can use any branch:
|
||||
|
||||
svn co https://svn.ietf.org/svn/tools/ietfdb/trunk
|
||||
cd trunk
|
||||
|
||||
3. **TEMPORARY:** Replace the contents of the `docker` directory with [Lars'
|
||||
files](https://svn.ietf.org/svn/tools/ietfdb/personal/lars/7.39.1.dev0/docker/).
|
||||
|
||||
4. **TEMPORARY:** Until [Lars'
|
||||
changes](https://svn.ietf.org/svn/tools/ietfdb/personal/lars/7.39.1.dev0/docker/)
|
||||
have been merged and a docker image is available for download, you will need
|
||||
to build it locally:
|
||||
|
||||
docker/build
|
||||
|
||||
This will take a while, but only needs to be done once.
|
||||
|
||||
5. Use the `docker/run` script to start the datatracker container. You will be
|
||||
dropped into a shell from which you can start the datatracker and execute
|
||||
related commands as usual, for example
|
||||
|
||||
ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
to start the datatracker.
|
||||
|
||||
You can also pass additional arguments to `docker/run`, in which case they
|
||||
will be executed in the container (instead of a shell being started.)
|
||||
|
||||
If you do not already have a copy of the IETF database available in the
|
||||
`data` directory, one will be downloaded and imported the first time you run
|
||||
`docker/run`. This will take some time.
|
||||
|
||||
Once the datatracker has started, you should be able to open
|
||||
[http://localhost:8000](http://localhost:8000) in a browser and see the
|
||||
landing page.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- If the database fails to start, the cause is usually an incompatibility
|
||||
between the database that last touched the files in `data/mysql` and the
|
||||
database running inside the docker container.
|
||||
|
||||
The solution is to blow away your existing database (`rm -rf data/mysql`). A
|
||||
fresh copy will be retrieved and imported next time you do `docker/run`, which
|
||||
should resolve this issue.
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
|
||||
==============================================================================
|
||||
Datatracker Development in a Docker Container (beta)
|
||||
==============================================================================
|
||||
|
||||
|
||||
Intro
|
||||
=====
|
||||
|
||||
Docker_ is a toolkit which lets you package software together with its
|
||||
dependencies in lightweight containers, and run it in isolated virtual
|
||||
environments.
|
||||
|
||||
During and just after IETF-94 I've spent quite a bit of time setting up a
|
||||
docker image which provides the dependencies needed to run the datatracker,
|
||||
and it's now available for beta testing. Hopefully this should make it
|
||||
substantially easier to get started with datatracker development.
|
||||
|
||||
Steps
|
||||
=====
|
||||
|
||||
1. Set up Docker on your preferred platform. Official installers exist for
|
||||
many Linux flavours, OS X, Windows and Cloud services. Here's the full `List
|
||||
of Installation Instructions`_.
|
||||
|
||||
Docker containers require the services of an underlying Linux API, which
|
||||
means that on OS X and Windows, these have to be provided by a virtual
|
||||
machine which runs a minimal Linux image. The virtual machine used on
|
||||
non-Linux platforms is commonly VirtualBox. On Linux kernels with version
|
||||
3.8 or later, no virtual machine is needed, as the docker images can be
|
||||
fully supported with the native kernel services.
|
||||
|
||||
Please follow the Docker installations all the way through to successfully
|
||||
running the ``hello-world`` example in a terminal window ( ``$ docker run
|
||||
hello-world``).
|
||||
|
||||
|
||||
2. Check out your datatracker branch as usual, in a suitable directory.
|
||||
We'll assume ``~/src/dt/`` here, and assume you are ``'coder'``::
|
||||
|
||||
~/src/dt/ $ svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/coder/6.8.2.dev0
|
||||
|
||||
3. In the checked-out working copy, you'll find a ``docker/`` directory and a
|
||||
``data/`` directory at the top level. We're first going to set up a copy of
|
||||
the MySQL database files under the ``data/`` directory.
|
||||
|
||||
There is a command in the ``docker/`` directory, ``setupdb`` which will do
|
||||
this for you, or you can do it manually.
|
||||
|
||||
Either run::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker/setupdb
|
||||
|
||||
or do this step-by-step: fetch down a pre-built copy of the datatracker
|
||||
database, place it in the ``data`` directory, unpack it, and fix
|
||||
permissions::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ cd data
|
||||
~/src/dt/6.8.2.dev0/data/ $ wget https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
|
||||
~/src/dt/6.8.2.dev0/data/ $ tar xjf ietf_utf8.bin.tar.bz2
|
||||
~/src/dt/6.8.2.dev0/data/ $ chmod -R go+rwX mysql
|
||||
|
||||
|
||||
4. In the ``docker/`` directory you'll also find a wrapper script named
|
||||
``'run'``. We will be using the wrapper to run a pre-built docker image
|
||||
fetched from the docker hub::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker/run
|
||||
|
||||
This will pull down the latest docker ietf/datatracker-environment image,
|
||||
start it up with appropriate settings, map the internal ``/var/lib/mysql/``
|
||||
directory to the external ``data/mysql/`` directory where we placed the
|
||||
database, set up a python virtualenv for you, install some dependencies,
|
||||
and drop you in a bash shell where you can run the datatracker.
|
||||
|
||||
6. You are now ready to run the tests::
|
||||
|
||||
(virtual) $ ietf/manage.py test --settings=settings_sqlitetest
|
||||
|
||||
and then start the dev server::
|
||||
|
||||
(virtual) $ ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
Note the IP address ``0.0.0.0`` used to make the dev server bind to all
|
||||
addresses. The internal port 8000 has been mapped to port 8000 externally,
|
||||
too. In order to find the IP address of the VirtualBox, run ``'$
|
||||
docker-machine ip'`` *outside* the virtual environment::
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ docker-machine ip
|
||||
192.168.59.103
|
||||
|
||||
~/src/dt/6.8.2.dev0/ $ open http://192.168.59.103:8000/
|
||||
|
||||
.. _Docker: https://www.docker.com/
|
||||
.. _`List of Installation Instructions`: https://docs.docker.com/v1.8/installation/
|
||||
.. _VirtualBox: https://www.virtualbox.org/
|
||||
|
||||
|
||||
110
docker/build
110
docker/build
|
|
@ -1,126 +1,92 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.20
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
export LANG=C
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$program - Build a docker datatracker image.
|
||||
$program - Build a datatracker docker image
|
||||
|
||||
SYNOPSIS
|
||||
$program [OPTIONS] ARGS
|
||||
$program [OPTIONS]
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script builds a debian-based docker image which has been
|
||||
This script builds a Ubuntu-based docker image that has been
|
||||
set up with the dependencies needed to easily run the IETF
|
||||
datatracker in development mode. It uses docker/Dockerfile;
|
||||
i.e., the Dockerfile in the same directory as this script.
|
||||
It assumes that the user has upload rights for the docker
|
||||
ietf/datatracker-environment repository, in order to push the
|
||||
image.
|
||||
|
||||
OPTIONS
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Written by:
|
||||
Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Lars Eggert, <lars@eggert.org>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
EOF
|
||||
the code. All rights reserved. Redistribution and use in source and
|
||||
binary forms, with or without modification, is permitted pursuant to,
|
||||
and subject to the license terms contained in, the Revised BSD
|
||||
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
|
||||
Relating to IETF Documents(https://trustee.ietf.org/license-info).
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=hlt:vV
|
||||
longopts=help,local,tag=,verbose,version
|
||||
|
||||
# Default values
|
||||
BRANCH=$(svn log -v ^/tags -l 2 | grep 'A /tags/[1-9]' | awk '{print $2}')
|
||||
TAG=${BRANCH##*/}
|
||||
LOCAL=""
|
||||
IMAGE=ietf/datatracker-environment
|
||||
TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}' | tr -d '\r')")
|
||||
LOCAL=1
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
# Option parsing
|
||||
shortopts=hut:V
|
||||
args=$(getopt -o$shortopts $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-l| --local) LOCAL=1;; # Don't upload
|
||||
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the latest svn tags name
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
-h) usage; exit;; # Show this help, then exit
|
||||
-u) LOCAL=0;; # Upload image to repository after build
|
||||
-t) TAG=$2; shift;; # Use this docker image tag
|
||||
-V) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
if [ "$(uname)" == "Linux" ]; then
|
||||
die "Didn't expect to run this script on Linux -- are you inside docker?"
|
||||
fi
|
||||
|
||||
docker rmi -f ietf/datatracker-environment:trunk || true
|
||||
docker build -t ietf/datatracker-environment:$TAG docker/
|
||||
docker tag $(docker images -q ietf/datatracker-environment | head -n 1) ietf/datatracker-environment:latest
|
||||
docker rmi -f $IMAGE:trunk 2>/dev/null || true
|
||||
docker build --progress plain -t "$IMAGE:$TAG" docker/
|
||||
docker tag "$(docker images -q $IMAGE | head -n 1)" $IMAGE:latest
|
||||
if [ -z "$LOCAL" ]; then
|
||||
docker push ietf/datatracker-environment:latest
|
||||
docker push ietf/datatracker-environment:$TAG
|
||||
docker push $IMAGE:latest
|
||||
docker push "$IMAGE:$TAG"
|
||||
fi
|
||||
|
|
@ -1,118 +1,89 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.11
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
export LANG=C
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$program - Make a tarball of the MySQL database files and upload it.
|
||||
$program - Make a tarball of the MySQL database files and upload it
|
||||
|
||||
SYNOPSIS
|
||||
$program [OPTIONS]
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script creates a compressed tarball from the MySQL database files
|
||||
on disk, and uploads it to the ietf datatracker developer area on
|
||||
www.ietf.org.
|
||||
|
||||
It is intended to be used with the docker datatracker environment, after
|
||||
you have set up the database with docker/setupdb, started the docker
|
||||
image with docker/run, and updated the database with docker/updatedb.
|
||||
image with docker/run, and updated the database with docker/updatedb.
|
||||
|
||||
To use it, exit from the docker container, to make sure that mysqldb
|
||||
isn't running and all the files consistent and available for copy. Then
|
||||
run docker/$program outside the docker container. You need to have ssh
|
||||
access to www.ietf.org in order for the scp command to succeed.
|
||||
|
||||
OPTIONS
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Written by:
|
||||
Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Lars Eggert, <lars@eggert.org>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
EOF
|
||||
the code. All rights reserved. Redistribution and use in source and
|
||||
binary forms, with or without modification, is permitted pursuant to,
|
||||
and subject to the license terms contained in, the Revised BSD
|
||||
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
|
||||
Relating to IETF Documents(https://trustee.ietf.org/license-info).
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=hvV
|
||||
longopts=help,verbose,version
|
||||
|
||||
# Default values
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
shortopts=hV
|
||||
args=$(getopt -o$shortopts $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
-h) usage; exit;; # Show this help, then exit
|
||||
-V) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
if [ -e "/.dockerenv" -o -n "$(grep -s '/docker/' /proc/self/cgroup)" ]; then
|
||||
die "It looks as if you're running inside docker -- please quit docker first."
|
||||
fi
|
||||
|
|
@ -123,4 +94,4 @@ cd $workdir
|
|||
echo "Building tarfile ..."
|
||||
tar cjf ietf_utf8.bin.tar.bz2 mysql
|
||||
echo "Copying tarfile to ietfa.amsl.com ..."
|
||||
scp ietf_utf8.bin.tar.bz2 ietfa.amsl.com:/a/www/www6s/lib/dt/sprint/
|
||||
scp ietf_utf8.bin.tar.bz2 ietfa.amsl.com:/a/www/www6s/lib/dt/sprint/
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$progdir/run ietf/manage.py runserver 0.0.0.0:8000
|
||||
|
||||
|
||||
|
|
@ -1,195 +1,91 @@
|
|||
#!/bin/bash
|
||||
|
||||
# A little bit of setup
|
||||
export LANG=en_GB.UTF-8
|
||||
MYSQLDIR=/var/lib/mysql
|
||||
|
||||
echo "Gathering info ..."
|
||||
MYSQLDIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
|
||||
if [ ! "$USER" ]; then
|
||||
echo "Environment variable USER is not set -- will set USER='django'."
|
||||
USER="django"
|
||||
fi
|
||||
if [ ! "$UID" ]; then
|
||||
echo "Environment variable UID is not set -- will set UID='1000'."
|
||||
UID="1000"
|
||||
fi
|
||||
if [ ! "$GID" ]; then
|
||||
echo "Environment variable GID is not set -- will set GID='1000'."
|
||||
GID="1000"
|
||||
fi
|
||||
if [ ! "$TAG" ]; then
|
||||
echo "Environment variable TAG is not set -- will set TAG='datatracker'."
|
||||
TAG="datatracker"
|
||||
fi
|
||||
echo "User $USER ($UID:$GID)"
|
||||
|
||||
echo "Checking if MySQL base data exists ..."
|
||||
if [ ! -d $MYSQLDIR/mysql ]; then
|
||||
echo "WARNING: Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
|
||||
if [ ! -d "$MYSQLDIR" ]; then
|
||||
echo "WARNING: Expected the directory $MYSQLDIR to exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Setting up the 'mysql' user for database file access ..."
|
||||
MYSQL_TARGET_GID=$(stat -c "%g" $MYSQLDIR/mysql)
|
||||
if ! grep -q ":$MYSQL_TARGET_GID:$" /etc/group; then
|
||||
groupadd -g $MYSQL_TARGET_GID mysqldata
|
||||
fi
|
||||
usermod -a -G $MYSQL_TARGET_GID mysql
|
||||
service rsyslog start
|
||||
|
||||
echo "Checking if MySQL is running ..."
|
||||
if ! /etc/init.d/mysql status; then
|
||||
echo "Starting mysql ..."
|
||||
/etc/init.d/mysql start
|
||||
if [ -z "$(ls -A $MYSQLDIR/mysql 2>/dev/null)" ]; then
|
||||
can=$(date -r /mysql +%s)
|
||||
now=$(date +%s)
|
||||
age=$((($now - $can)/86400))
|
||||
echo "NOTE: Database empty; populating it from canned snapshot ($age days old)"
|
||||
echo " This will take a little while..."
|
||||
cp -r /mysql/* $MYSQLDIR
|
||||
fi
|
||||
|
||||
echo "Checking if syslogd is running ..."
|
||||
if ! /etc/init.d/rsyslog status > /dev/null; then
|
||||
echo "Starting syslogd ..."
|
||||
/etc/init.d/rsyslog start
|
||||
service mariadb start
|
||||
|
||||
if ! service mariadb status; then
|
||||
echo "ERROR: MySQL didn't start. Here are some possible causes:"
|
||||
echo "-------------------------------------------------------------------"
|
||||
grep mysqld /var/log/syslog
|
||||
echo "-------------------------------------------------------------------"
|
||||
echo "Such errors are usually due to a corrupt or outdated database."
|
||||
echo "Remove your local database and let the image install a clean copy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Give debian-sys-maint access, to avoid complaints later
|
||||
mysql mysql <<< "GRANT ALL PRIVILEGES on *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '$(awk '/^password/ {print $3; exit}' /etc/mysql/debian.cnf )' WITH GRANT OPTION; FLUSH PRIVILEGES;"
|
||||
|
||||
echo "Checking if the IETF database exists at $MYSQLDIR ..."
|
||||
if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
|
||||
if [ -z "$DATADIR" ]; then
|
||||
echo "DATADIR is not set, but the IETF database needs to be set up -- can't continue, exiting the docker init script."
|
||||
exit 1
|
||||
fi
|
||||
ls -l $MYSQLDIR
|
||||
|
||||
if ! /etc/init.d/mysql status; then
|
||||
echo "Didn't find the IETF database, but can't set it up either, as MySQL isn't running."
|
||||
else
|
||||
echo "Creating database ..."
|
||||
mysqladmin -u root --default-character-set=utf8 create ietf_utf8
|
||||
|
||||
echo "Setting up permissions ..."
|
||||
mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
|
||||
|
||||
echo "Fetching database ..."
|
||||
DUMPDIR=/home/$USER/$DATADIR
|
||||
wget -N -P $DUMPDIR http://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz
|
||||
|
||||
echo "Loading database ..."
|
||||
gunzip < $DUMPDIR/ietf_utf8.sql.gz \
|
||||
| pv --progress --bytes --rate --eta --cursor --size $(gzip --list --quiet $DUMPDIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| mysql --user=django --password=RkTkDPFnKpko -s -f ietf_utf8 \
|
||||
&& rm /tmp/ietf_utf8.sql.gz
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! grep -q ":$GID:$" /etc/group ; then
|
||||
echo "Creating group entry for GID '$GID' ..."
|
||||
groupadd -g "$GID" "$USER"
|
||||
fi
|
||||
if ! id -u "$USER" &> /dev/null; then
|
||||
echo "Creating user '$USER' ..."
|
||||
useradd -s /bin/bash --groups staff,sudo --uid $UID --gid $GID $USER
|
||||
echo "$USER:$USER" | chpasswd
|
||||
fi
|
||||
|
||||
VIRTDIR="/opt/home/$USER/$TAG"
|
||||
echo "Checking that there's a virtual environment for $TAG ..."
|
||||
if [ ! -f $VIRTDIR/bin/activate ]; then
|
||||
echo "Setting up python virtualenv at $VIRTDIR ..."
|
||||
mkdir -p $VIRTDIR
|
||||
python3.6 -m venv $VIRTDIR
|
||||
echo -e "
|
||||
# This is from $VIRTDIR/bin/activate, to activate the
|
||||
# datatracker virtual python environment on docker container entry:
|
||||
" >> /etc/bash.bashrc
|
||||
cat $VIRTDIR/bin/activate >> /etc/bash.bashrc
|
||||
cat /usr/local/share/datatracker/setprompt >> /etc/bash.bashrc
|
||||
else
|
||||
echo "Using virtual environment at $VIRTDIR"
|
||||
fi
|
||||
|
||||
echo "Activating the virtual python environment ..."
|
||||
. $VIRTDIR/bin/activate
|
||||
|
||||
|
||||
if ! $VIRTDIR/bin/python -c "import django"; then
|
||||
echo "Installing requirements ..."
|
||||
pip install --upgrade pip
|
||||
reqs=/home/$USER/$CWD/requirements.txt
|
||||
if [ ! -f $reqs ]; then
|
||||
echo " Using $reqs"
|
||||
pip install -r $reqs
|
||||
else
|
||||
echo " Didn't find $reqs"
|
||||
echo " Using /usr/local/share/datatracker/requirements.txt"
|
||||
pip install -r /usr/local/share/datatracker/requirements.txt
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f /home/$USER/$CWD/ietf/settings_local.py ]; then
|
||||
if [ ! -f /root/src/ietf/settings_local.py ]; then
|
||||
echo "Setting up a default settings_local.py ..."
|
||||
cp /home/$USER/$CWD/docker/settings_local.py /home/$USER/$CWD/ietf/settings_local.py
|
||||
cp /root/src/docker/settings_local.py /root/src/ietf/settings_local.py
|
||||
fi
|
||||
|
||||
for sub in test/id/ test/staging/ test/archive/ test/rfc test/media test/wiki/ietf; do
|
||||
dir="/home/$USER/$CWD/$sub"
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Creating dir $dir"
|
||||
mkdir -p "$dir";
|
||||
fi
|
||||
done
|
||||
|
||||
for sub in \
|
||||
nomcom_keys/public_keys \
|
||||
developers/ietf-ftp \
|
||||
developers/ietf-ftp/bofreq \
|
||||
developers/ietf-ftp/charter \
|
||||
developers/ietf-ftp/conflict-reviews \
|
||||
developers/ietf-ftp/internet-drafts \
|
||||
developers/ietf-ftp/rfc \
|
||||
developers/ietf-ftp/status-changes \
|
||||
developers/ietf-ftp/yang/catalogmod \
|
||||
developers/ietf-ftp/yang/draftmod \
|
||||
developers/ietf-ftp/yang/ianamod \
|
||||
developers/ietf-ftp/yang/invalmod \
|
||||
developers/ietf-ftp/yang/rfcmod \
|
||||
developers/www6s \
|
||||
developers/www6s/staging \
|
||||
developers/www6s/wg-descriptions \
|
||||
developers/www6s/proceedings \
|
||||
developers/www6/ \
|
||||
developers/www6/iesg \
|
||||
developers/www6/iesg/evaluation \
|
||||
test/id \
|
||||
test/staging \
|
||||
test/archive \
|
||||
test/rfc \
|
||||
test/media \
|
||||
test/wiki/ietf \
|
||||
data/nomcom_keys/public_keys \
|
||||
data/developers/ietf-ftp \
|
||||
data/developers/ietf-ftp/bofreq \
|
||||
data/developers/ietf-ftp/charter \
|
||||
data/developers/ietf-ftp/conflict-reviews \
|
||||
data/developers/ietf-ftp/internet-drafts \
|
||||
data/developers/ietf-ftp/rfc \
|
||||
data/developers/ietf-ftp/status-changes \
|
||||
data/developers/ietf-ftp/yang/catalogmod \
|
||||
data/developers/ietf-ftp/yang/draftmod \
|
||||
data/developers/ietf-ftp/yang/ianamod \
|
||||
data/developers/ietf-ftp/yang/invalmod \
|
||||
data/developers/ietf-ftp/yang/rfcmod \
|
||||
data/developers/www6s \
|
||||
data/developers/www6s/staging \
|
||||
data/developers/www6s/wg-descriptions \
|
||||
data/developers/www6s/proceedings \
|
||||
data/developers/www6/ \
|
||||
data/developers/www6/iesg \
|
||||
data/developers/www6/iesg/evaluation \
|
||||
; do
|
||||
dir="/home/$USER/$CWD/data/$sub"
|
||||
dir="/root/src/$sub"
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo "Creating dir $dir"
|
||||
echo "Creating dir $dir"
|
||||
mkdir -p "$dir";
|
||||
chown "$USER" "$dir"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ! -f "/home/$USER/$CWD/test/data/draft-aliases" ]; then
|
||||
echo "Generating draft aliases ..."
|
||||
ietf/bin/generate-draft-aliases }
|
||||
fi
|
||||
python -m smtpd -n -c DebuggingServer localhost:2025 &
|
||||
echo
|
||||
|
||||
if [ ! -f "/home/$USER/$CWD/test/data/group-aliases" ]; then
|
||||
echo "Generating group aliases ..."
|
||||
ietf/bin/generate-wg-aliases }
|
||||
fi
|
||||
|
||||
chown -R $USER /opt/home/$USER
|
||||
chmod -R g+w /usr/local/lib/ # so we can patch libs if needed
|
||||
|
||||
cd "/home/$USER/$CWD" || cd "/home/$USER/"
|
||||
|
||||
echo "Done!"
|
||||
if ! echo "$LANG" | grep "UTF-8"; then
|
||||
echo ""
|
||||
echo "Make sure you export LANG=en_GB.UTF-8 (or another UTF-8 locale) in your .bashrc"
|
||||
if [ -z "$*" ]; then
|
||||
echo "You can execute arbitrary commands now, e.g.,"
|
||||
echo
|
||||
echo " ietf/manage.py runserver 0.0.0.0:8000"
|
||||
echo
|
||||
echo "to start a development instance of the Datatracker."
|
||||
echo
|
||||
bash
|
||||
else
|
||||
echo "LANG=$LANG"
|
||||
echo "Executing \"$*\" and stopping container."
|
||||
echo
|
||||
bash -c "$*"
|
||||
fi
|
||||
|
||||
HOME=/opt/home/$USER
|
||||
|
||||
su -p $USER
|
||||
service mariadb stop
|
||||
service rsyslog stop
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
sudo su - -c "apt-get update \
|
||||
&& apt-get install -qy \
|
||||
apt-file \
|
||||
apt-show-versions \
|
||||
graphviz \
|
||||
ghostscript \
|
||||
apache2-utils \
|
||||
chromium-driver \
|
||||
enscript \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*"
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$progdir/run python -m smtpd -n -c DebuggingServer localhost:2025
|
||||
|
|
@ -1,127 +1,105 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.11
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
export LANG=C
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$program - Copy additional data files from the ietf server
|
||||
|
||||
SYNOPSIS
|
||||
$program [OPTIONS]
|
||||
$program [OPTIONS] [DESTINATION]
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script copies additional data files used by the datatracker
|
||||
from the ietf server to a local directory, for instance drafts,
|
||||
charters, rfcs, agendas, minutes, etc.
|
||||
from the ietf server to a local directory, for instance drafts,
|
||||
charters, rfcs, agendas, minutes, etc.
|
||||
|
||||
If no destination is given, the default is data/developers.
|
||||
|
||||
OPTIONS
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Written by:
|
||||
Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Lars Eggert, <lars@eggert.org>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
EOF
|
||||
the code. All rights reserved. Redistribution and use in source and
|
||||
binary forms, with or without modification, is permitted pursuant to,
|
||||
and subject to the license terms contained in, the Revised BSD
|
||||
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
|
||||
Relating to IETF Documents(https://trustee.ietf.org/license-info).
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=hvV
|
||||
longopts=help,verbose,version
|
||||
|
||||
# Default values
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
args=$(getopt -o$shortopts $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
-h) usage; exit;; # Show this help, then exit
|
||||
-v) VERBOSE=1;; # Be more talkative
|
||||
-V) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
if [ $# -lt 1 ]; then
|
||||
DEST_ROOT=data/developers
|
||||
else
|
||||
DEST_ROOT="${1%/}"
|
||||
fi
|
||||
echo "Using destination $DEST_ROOT"
|
||||
|
||||
[ $# -lt 1 ] && die "Missing argument: rsync destination"
|
||||
|
||||
DEST_ROOT="${1%/}"
|
||||
for dir in charter conflict-reviews internet-drafts review rfc slides status-changes yang; do
|
||||
dest="$DEST_ROOT/ietf-ftp/$dir"
|
||||
mkdir -p "$dest"
|
||||
echo "Fetching $dest ..."
|
||||
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::everything-ftp/$dir/ $dest/
|
||||
done
|
||||
|
||||
for dir in floor photo; do
|
||||
dest="$DEST_ROOT/media/$dir"
|
||||
mkdir -p "$dest"
|
||||
echo "Fetching $dest ..."
|
||||
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::dev.media/$dir/ $dest/
|
||||
done
|
||||
|
||||
|
||||
dest="$DEST_ROOT/archive/id"
|
||||
echo "Fetching $dest ..."
|
||||
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::id-archive/ $dest/
|
||||
|
||||
|
||||
dest="$DEST_ROOT/archive/id"
|
||||
mkdir -p "$dest"
|
||||
echo "Fetching $dest ..."
|
||||
rsync -auz ${VERBOSE:+--info=progress2} rsync.ietf.org::id-archive/ $dest/
|
||||
277
docker/run
277
docker/run
|
|
@ -1,15 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
version=0.20
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo $parent | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
|
|
@ -19,269 +19,92 @@ SYNOPSIS
|
|||
$program [OPTIONS] ARGS
|
||||
|
||||
DESCRIPTION
|
||||
This is a wrapper which runs an Ubuntu-based docker image which
|
||||
has been set up with the dependencies needed to easily run the
|
||||
IETF datatracker in development mode.
|
||||
|
||||
This is a wrapper which runs docker with suitable arguments on a
|
||||
debian-based docker image which has been set up with the dependencies
|
||||
needed to easily run the IETF datatracker in development mode. By
|
||||
default, it expects to find the MySQL database files at
|
||||
$parent/data/mysql, which is mapped inside the
|
||||
container to /var/lib/mysql, and it will set up a home directory for
|
||||
the current user ($USER) and map it to $HOME.
|
||||
MySQL database files at data/mysql will be used; if they do not exist,
|
||||
a database dump will be retrieved and restored on first run.
|
||||
|
||||
OPTIONS
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Written by:
|
||||
Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Lars Eggert, <lars@eggert.org>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. Redistribution and use in source and
|
||||
binary forms, with or without modification, is permitted pursuant to,
|
||||
and subject to the license terms contained in, the Revised BSD
|
||||
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
|
||||
Relating to IETF Documents(https://trustee.ietf.org/license-info).
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=dhi:m:Mp:r:t:vVu:cC
|
||||
longopts=download-data,help,ietfdb-url=,mysqldata=,no-mysqldir,port=,docker-repo=,tag=,verbose,version,user=,cached,no-cached
|
||||
|
||||
# Default values
|
||||
MYSQLDIR=$parent/data/mysql
|
||||
FILEDIR=$parent/../data/
|
||||
PORT=8000
|
||||
REPO="ietf/datatracker-environment"
|
||||
DBURL=https://www.ietf.org/lib/dt/sprint/ietf_utf8.bin.tar.bz2
|
||||
WHO=$(whoami)
|
||||
CACHED=''
|
||||
CACHED=':cached'
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
if [ "$(uname)" = "Darwin" ]; then
|
||||
CACHED=':cached'
|
||||
fi
|
||||
fi
|
||||
# Option parsing
|
||||
shortopts=cChp:V
|
||||
args=$(getopt -o$shortopts $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-c| --cached) CACHED=':cached';; # Use cached disk access to reduce system load
|
||||
-C| --no-cached) CACHED='';; # Use fully synchronized disk access
|
||||
-d| --download-data) DOWNLOAD=1;; # Download and set up the database files
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-f| --filedir) FILEDIR=$2; shift;; # Set the desired location of drafts, charters etc.
|
||||
-i| --ietfdb-url) DBURL=$2; shift;; # Use an alternative database tarball URL
|
||||
-m| --mysqldir) MYSQLDIR=$2; shift;; # Set the desired location of MySQL's database files
|
||||
-p| --port) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
|
||||
-r| --docker-repo) REPO=$2; shift;; # Use the given docker repository, instead of the default
|
||||
-t| --tag) TAG=$2; shift;; # Use this docker image tag, instead of the svn branch name
|
||||
-u| --user) WHO=$2; shift;; # Run the container as the specified user
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
-c) CACHED=':cached';; # Use cached disk access to reduce system load
|
||||
-C) CACHED=':consistent';; # Use fully synchronized disk access
|
||||
-h) usage; exit;; # Show this help, then exit
|
||||
-p) PORT=$2; shift;; # Bind the container's port 8000 to external port PORT
|
||||
-V) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
TAG=$(basename $(svn info $parent | grep ^URL | awk '{print $2}'))
|
||||
TAG=$(basename "$(svn info "$parent" | grep ^URL | awk '{print $2}' | tr -d '\r')")
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
[ -f /proc/1/cgroups ] && grep 'docker' /proc/1/cgroups && die "It looks like you're inside docker already ..."
|
||||
|
||||
if [ "$(uname)" = "Darwin" ]; then
|
||||
APP="/Applications/Docker.app"
|
||||
CMD="open -a"
|
||||
elif [ "$(uname)" = "Linux" ]; then
|
||||
echo "Running on Linux."
|
||||
elif [[ $(uname) =~ CYGWIN.* ]]; then
|
||||
echo "Running under Cygwin."
|
||||
CMD="echo"
|
||||
MYSQLDIR=$(echo $MYSQLDIR | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
|
||||
WHO=$(echo $WHO | sed -e 's/^.*\\//' | tr -d \\r)
|
||||
DRIVE=$(echo $USERPROFILE | sed -e 's/\(.\).*/\1/' -e 'y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefdhijklmnopqartuvwxyz/')
|
||||
HOME=$DRIVE$(echo $USERPROFILE | sed -e 's/\\/\//g' -e's/.\(.*\)/\1/')
|
||||
echo "Using home dir $HOME"
|
||||
PWD=$(echo $PWD | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
|
||||
if [ "${PWD#$HOME}" = "$PWD" ]; then
|
||||
die "You must work inside your home directory ($HOME)"
|
||||
fi
|
||||
CGWPARENT=$(echo $parent | sed -e 's/^\(.\)\:/\/cygdrive\/\1/')
|
||||
ICSFILES=$(/usr/bin/find $CGWPARENT/vzic/zoneinfo/ -name '*.ics' -print)
|
||||
for ICSFILE in $ICSFILES
|
||||
do
|
||||
LINK=$(head -n1 $ICSFILE | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then
|
||||
echo "Running under Cygwin, replacing symlinks with file copies"
|
||||
ICSFILES=$(/usr/bin/find "$parent/vzic/zoneinfo/" -name '*.ics' -print)
|
||||
for ICSFILE in $ICSFILES; do
|
||||
LINK=$(head -n1 "$ICSFILE" | sed -e '/link .*/!d' -e 's/link \(.*\)/\1/')
|
||||
if [ "$LINK" ]; then
|
||||
WDIR=$(dirname $ICSFILE)
|
||||
echo "Replacing $(basename $ICSFILE) with $LINK"
|
||||
cp -f $WDIR/$LINK $ICSFILE
|
||||
WDIR=$(dirname "$ICSFILE")
|
||||
echo "Replacing $(basename "$ICSFILE") with $LINK"
|
||||
cp -f "$WDIR/$LINK" "$ICSFILE"
|
||||
fi
|
||||
done
|
||||
else
|
||||
die "This script does not have support for your architecture ($(uname)); sorry :-("
|
||||
fi
|
||||
|
||||
WHOUID=$(id -u $WHO)
|
||||
WHOGID=$(id -g $WHO)
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
echo "Not trying to start a virtual docker machine on Linux"
|
||||
elif [[ $(uname) =~ CYGWIN.* ]]; then
|
||||
if ! docker info 1> /dev/null 2>&1; then
|
||||
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
|
||||
die "Don't know how to start docker when running under Cygwin"
|
||||
fi
|
||||
TAG=$(echo $TAG | tr -d \\r)
|
||||
URL="http://localhost:$PORT/"
|
||||
elif [ -e "$APP" ]; then
|
||||
info=$(docker info 2>&1 || true)
|
||||
if ! docker info 1> /dev/null 2>&1; then
|
||||
echo -e "The docker VM doesn't seem to be running; docker info gave:\n $info"
|
||||
echo "Will attempt to start docker by doing '\$ $CMD $APP':"
|
||||
$CMD $APP
|
||||
declare -i count
|
||||
printf "Waiting for docker engine .."
|
||||
while true; do
|
||||
printf "."
|
||||
sleep 2
|
||||
if docker info >/dev/null 2>&1; then break; fi
|
||||
count=$(( $count + 1))
|
||||
if [ $count -gt 10 ]; then
|
||||
die "No contact with docker engine; bailing out."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
URL="http://localhost:$PORT/"
|
||||
else
|
||||
if [ $($machine status default | tr "A-Z" "a-z" ) != "running" ]; then
|
||||
echo "The docker VM doesn't seem to be running; will attempt to start it by doing '\$ $machine start':"
|
||||
$machine start || die "Failed taking up the Docker VM"
|
||||
fi
|
||||
|
||||
if [ -f "$machine" ]; then
|
||||
if [ $($machine status default | tr "A-Z" "a-z") = "running" ]; then
|
||||
tmpfile=$(mktemp docker.run.XXXXXXXX)
|
||||
if $machine env 2>/dev/null | grep DOCKER_ > $tmpfile; then
|
||||
mv $tmpfile ~/.docker-info
|
||||
elif printenv | grep DOCKER_ > $tmpfile; then
|
||||
mv $tmpfile ~/.docker-info
|
||||
else
|
||||
rm $tmpfile
|
||||
die "Failed setting the appropriate DOCKER_* environment variables."
|
||||
fi
|
||||
. ~/.docker-info
|
||||
else
|
||||
rm -f ~/.docker-info
|
||||
fi
|
||||
ip=$($machine ip)
|
||||
URL="http://$ip:$PORT/"
|
||||
fi
|
||||
fi
|
||||
|
||||
image=$(docker ps | grep "$REPO:$TAG" | awk '{ print $1 }')
|
||||
if [ "$image" ]; then
|
||||
if [ "$*" ]; then
|
||||
echo "Running 'cd ~/${parent#$HOME/}; $*'"
|
||||
docker exec -u "$WHO" -t $image bash -i -c "cd; cd \"${parent#$HOME/}\"; $*"
|
||||
else
|
||||
docker exec -u "$WHO" -ti $image bash -i
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo "Starting a docker container for '$TAG'."
|
||||
|
||||
if [ -n "$DOWNLOAD" ]; then
|
||||
(
|
||||
cd "$(dirname $MYSQLDIR)"
|
||||
wget -N "$DBURL" && tar xjf ietf_utf8.bin.tar.bz2 && chmod -R go+rwX mysql
|
||||
)
|
||||
[ -d "$MYSQLDIR" ] || die "The download seems to have failed; still no $MYSQLDIR. Giving up."
|
||||
else
|
||||
[ -d "$MYSQLDIR" ] || die "Expected $MYSQLDIR to exist, but it\ndidn't. Use '$program -d' to download and unpack the database."
|
||||
fi
|
||||
|
||||
if ! docker images "$REPO" | grep -q "$TAG"; then
|
||||
echo "Fetching docker image '$REPO:$TAG'"
|
||||
if ! docker pull "$REPO:$TAG"; then
|
||||
docker pull "$REPO:latest" || die "Failed to pull down the '$REPO:latest' docker image"
|
||||
id=$(docker images "$REPO" | grep latest | awk '{print $3}')
|
||||
echo "Tagging $REPO:latest as $REPO:$TAG for use as environment for this branch."
|
||||
docker tag $id "$REPO:$TAG"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "\nThe web interface for 'runserver' should appear on $URL\n"
|
||||
echo -e "User $WHO ($WHOUID:$WHOGID)"
|
||||
if [ -z "$MYSQLDIR" ]; then
|
||||
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
|
||||
-e USER="$WHO" -e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" \
|
||||
-e TAG="$TAG" -e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
|
||||
"$REPO:$TAG" "$@"
|
||||
else
|
||||
docker run -ti -p $PORT:8000 -v "$HOME:/home/$WHO$CACHED" \
|
||||
-v "$MYSQLDIR:/var/lib/mysql" -e USER="$WHO" \
|
||||
-e DATADIR="${parent#$HOME/}/data" -e CWD="${PWD#$HOME/}" -e TAG="$TAG" \
|
||||
-e FILEDIR=${FILEDIR#$HOME} -e UID="$WHOUID" -e GID="$WHOGID" \
|
||||
"$REPO:$TAG" "$@"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Committing changes in the container to an image:"
|
||||
image=$( docker images -q $REPO:$TAG)
|
||||
latest=$(docker ps -lq -f "ancestor=$image")
|
||||
docker commit $latest $REPO:$TAG
|
||||
|
||||
echo ""
|
||||
echo "Cleaning up containers and images"
|
||||
docker rm $latest
|
||||
DANGLING=$( docker images -f dangling=true -q )
|
||||
if [ -n "$DANGLING" ]; then
|
||||
echo "Dangling images: $DANGLING"
|
||||
docker rmi -f $DANGLING
|
||||
fi
|
||||
|
||||
fi
|
||||
echo "Starting a docker container for '$REPO:$TAG'."
|
||||
mkdir -p "$MYSQLDIR"
|
||||
docker run -ti -p "$PORT":8000 -p 33306:3306 \
|
||||
-v "$parent:/root/src$CACHED" \
|
||||
-v "$MYSQLDIR:/var/lib/mysql:delegated" \
|
||||
"$REPO:$TAG" "$@"
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
|
||||
# set a fancy prompt
|
||||
PS_TIME="\A "
|
||||
|
||||
if [ "$(whoami)" = "root" ]; then
|
||||
PS_COLOUR='\[\033[1;31m\]'
|
||||
PS_END="\[\033[m\]"
|
||||
else
|
||||
PS_COLOUR="\[\033[1;34m\]"
|
||||
PS_END="\[\033[m\]"
|
||||
fi
|
||||
|
||||
if [ "$TERM" = xterm ]; then
|
||||
PS_XTERM="\[\033]0;\]\h:\w\007"
|
||||
else
|
||||
PS_XTERM=""
|
||||
fi
|
||||
COLOUR_RED='\[\033[1;31m\]'
|
||||
COLOUR_BLK='\[\033[1;30m\]'
|
||||
if [ "$SHLVL" -gt 1 ]; then
|
||||
PS_SHLVL="$(eval "printf '>%.0s' {2..$SHLVL}") "
|
||||
else
|
||||
PS_SHLVL=""
|
||||
fi
|
||||
|
||||
PS1="$PS_XTERM\n$COLOUR_RED$PS_SHLVL$PS_COLOUR$PS_TIME$COLOUR_RED$PS_COLOUR${VIRTUAL_ENV:+$COLOUR_RED($(basename $VIRTUAL_ENV))$PS_COLOUR }\w\n\u @ $COLOUR_BLK\h$PS_COLOUR \\$ $PS_END"
|
||||
114
docker/setupdb
114
docker/setupdb
|
|
@ -1,114 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
version=0.10
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
$program - Set up a local copy of the IETF database MySQL files
|
||||
|
||||
SYNOPSIS
|
||||
$program [OPTIONS] ARGS
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script downloads a prebuilt copy of the IETF database MySQL
|
||||
files, ready for mapping into the /var/lib/mysql/ directory of the
|
||||
ietf/database-environment Docker image.
|
||||
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=hvV
|
||||
longopts=help,verbose,version
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
[ -n "$MYSQLDIR" ] || MYSQLDIR=$parent/data/mysql
|
||||
[ -n "$URL" ] || URL=rsync.ietf.org::dev.db/ietf_utf8.bin.tar.bz2
|
||||
|
||||
cd $(dirname $MYSQLDIR)
|
||||
echo "Downloading a MySQL database image ..."
|
||||
set $(rsync --version | head -n1 | awk '{print $3}' | tr '.' ' ')
|
||||
maj=$1; min=$2; fix=$3
|
||||
if [ "$maj" -ge "3" -a "$min" -ge "1" ]; then
|
||||
rsync --info=progress2 $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
|
||||
else
|
||||
rsync -v $URL ./ && echo -e "\nUnpacking database ..." && tar xjf ietf_utf8.bin.tar.bz2 && echo "Fixing permissions ..." && chmod -R go+rwX mysql
|
||||
fi
|
||||
137
docker/updatedb
137
docker/updatedb
|
|
@ -1,15 +1,15 @@
|
|||
#!/bin/bash
|
||||
# -*- indent-with-tabs: 1 -*-
|
||||
|
||||
version=0.10
|
||||
version=0.20
|
||||
program=${0##*/}
|
||||
progdir=${0%/*}
|
||||
if [ "$progdir" = "$program" ]; then progdir="."; fi
|
||||
if [ "$progdir" = "." ]; then progdir="$PWD"; fi
|
||||
parent=$(dirname $progdir)
|
||||
parent=$(dirname "$progdir")
|
||||
if [ "$parent" = "." ]; then parent="$PWD"; fi
|
||||
if [[ $(uname) =~ CYGWIN.* ]]; then parent=$(echo "$parent" | sed -e 's/^\/cygdrive\/\(.\)/\1:/'); fi
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function usage() {
|
||||
cat <<EOF
|
||||
NAME
|
||||
|
|
@ -19,140 +19,87 @@ SYNOPSIS
|
|||
$program [OPTIONS] ARGS
|
||||
|
||||
DESCRIPTION
|
||||
|
||||
This script downloads a dump of the IETF database and loads into the
|
||||
local sql server if it is newer than the current dump.
|
||||
|
||||
OPTIONS
|
||||
EOF
|
||||
echo -e "OPTIONS"
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
|
||||
else
|
||||
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
|
||||
fi
|
||||
grep -E '^\s+-[a-zA-Z])' "$0" | sed -E -e 's/\)[^#]+#/ /'
|
||||
cat <<EOF
|
||||
|
||||
FILES
|
||||
|
||||
AUTHOR
|
||||
Written by Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Written by:
|
||||
Henrik Levkowetz, <henrik@levkowetz.com>
|
||||
Lars Eggert, <lars@eggert.org>
|
||||
|
||||
COPYRIGHT
|
||||
|
||||
Copyright (c) 2015 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. License 'Simplified BSD', as specified
|
||||
in http://opensource.org/licenses/BSD-3-Clause.
|
||||
|
||||
Copyright (c) 2016 IETF Trust and the persons identified as authors of
|
||||
the code. All rights reserved. Redistribution and use in source and
|
||||
binary forms, with or without modification, is permitted pursuant to,
|
||||
and subject to the license terms contained in, the Revised BSD
|
||||
License set forth in Section 4.c of the IETF Trust’s Legal Provisions
|
||||
Relating to IETF Documents(https://trustee.ietf.org/license-info).
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
function die() {
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
echo -e "\n$program: error: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function note() {
|
||||
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
function version() {
|
||||
echo -e "$program $version"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Option parsing
|
||||
|
||||
# Options
|
||||
shortopts=DLZhqvV
|
||||
longopts=no-download,no-load,no-zap,help,quiet,verbose,version
|
||||
|
||||
shortopts=DLZhV
|
||||
LOAD=1
|
||||
DOWNLOAD=1
|
||||
DROP=1
|
||||
QUIET=""
|
||||
|
||||
if [ "$(uname)" = "Linux" ]; then
|
||||
args=$(getopt -o "$shortopts" --long "$longopts" -n "$program" -- $SV "$@")
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
eval set -- "$args"
|
||||
sed="sed -r"
|
||||
else
|
||||
# Darwin, BSDs
|
||||
args=$(getopt -o$shortopts $SV $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
sed="sed -E"
|
||||
fi
|
||||
args=$(getopt -o$shortopts $*)
|
||||
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
|
||||
set -- $args
|
||||
|
||||
while true ; do
|
||||
case "$1" in
|
||||
-D| --no-download) DOWNLOAD="";; #Don't download, use existing file
|
||||
-L| --no-load) LOAD=""; ;; # Don't load the database
|
||||
-Z| --no-zap) DROP="";; # Don't drop new tables
|
||||
-h| --help) usage; exit;; # Show this help, then exit
|
||||
-q| --quiet) QUIET="-q";; # Don't show progress
|
||||
-v| --verbose) VERBOSE=1;; # Be more talkative
|
||||
-V| --version) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
-D) DOWNLOAD="";; # Don't download, use existing file
|
||||
-L) LOAD=""; ;; # Don't load the database
|
||||
-Z) DROP="";; # Don't drop new tables
|
||||
-h) usage; exit;; # Show this help, then exit
|
||||
-V) version; exit;; # Show program version, then exit
|
||||
--) shift; break;;
|
||||
*) die "Internal error, inconsistent option specification: '$1'";;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# The program itself
|
||||
|
||||
echo "Gathering info ..."
|
||||
MYSQLDIR="$(/usr/sbin/mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
|
||||
MYSQLDIR=${MYSQLDIR%/}
|
||||
DATADIR=$parent/data
|
||||
|
||||
# echo "Checking if MySQL base data exists ..."
|
||||
# if [ ! -d $MYSQLDIR/mysql ]; then
|
||||
# die "Expected the directory $MYSQLDIR/mysql/ to exist -- have you downloaded and unpacked the IETF binary database tarball?"
|
||||
# fi
|
||||
|
||||
|
||||
# echo "Checking if the IETF database exists at $MYSQLDIR ..."
|
||||
# if [ ! -d $MYSQLDIR/ietf_utf8 ]; then
|
||||
# echo "Creating database ..."
|
||||
# mysqladmin -u root --default-character-set=utf8 create ietf_utf8
|
||||
#
|
||||
# echo "Setting up permissions ..."
|
||||
# mysql -u root ietf_utf8 <<< "GRANT ALL PRIVILEGES ON ietf_utf8.* TO django@localhost IDENTIFIED BY 'RkTkDPFnKpko'; FLUSH PRIVILEGES;"
|
||||
# fi
|
||||
|
||||
DUMP=ietf_utf8.sql.gz
|
||||
if [ "$DOWNLOAD" ]; then
|
||||
echo "Fetching database dump ..."
|
||||
wget $QUIET -N -P $DATADIR https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz || die "No new data, quitting."
|
||||
echo "Fetching database dump..."
|
||||
rsync --info=progress2 rsync.ietf.org::dev.db/$DUMP "$DATADIR"
|
||||
fi
|
||||
|
||||
if [ "$LOAD" ]; then
|
||||
echo "Loading database ..."
|
||||
if [ -z "$QUIET" ]; then
|
||||
gunzip < $DATADIR/ietf_utf8.sql.gz \
|
||||
| pv --progress --bytes --rate --eta --size $(gzip --list --quiet $DATADIR/ietf_utf8.sql.gz | awk '{ print $2 }') \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| $parent/ietf/manage.py dbshell
|
||||
else
|
||||
gunzip < $DATADIR/ietf_utf8.sql.gz \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| $parent/ietf/manage.py dbshell
|
||||
fi
|
||||
|
||||
echo "Loading database..."
|
||||
SIZE=$(pigz --list "$DATADIR/$DUMP" | tail -n 1 | awk '{ print $2 }')
|
||||
pigz -d < "$DATADIR/$DUMP" \
|
||||
| pv --progress --bytes --rate --eta --size "$SIZE" \
|
||||
| sed -e 's/ENGINE=MyISAM/ENGINE=InnoDB/' \
|
||||
| "$parent/ietf/manage.py" dbshell
|
||||
fi
|
||||
|
||||
if [ "$DROP" ]; then
|
||||
echo "Dropping tables not in the dump (so migrations can succeed) ..."
|
||||
diff <(zcat $DATADIR/ietf_utf8.sql.gz | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
|
||||
<($parent/ietf/manage.py dbshell <<< 'show tables;' | tail -n +2) \
|
||||
echo "Dropping tables not in the dump (so migrations can succeed)..."
|
||||
diff <(pigz -d -c "$DATADIR/$DUMP" | grep '^DROP TABLE IF EXISTS' | tr -d '`;' | awk '{ print $5 }') \
|
||||
<("$parent/ietf/manage.py" dbshell <<< 'show tables;' | tail -n +2) \
|
||||
| grep '^>' | awk '{print "drop table if exists", $2, ";";}' \
|
||||
| tee >(cat >&2) | $parent/ietf/manage.py dbshell
|
||||
| tee >(cat >&2) | "$parent/ietf/manage.py" dbshell
|
||||
fi
|
||||
Loading…
Reference in a new issue