diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 025442f2a..93f94ff91 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -48,13 +48,24 @@ "sqltools.connections": [ // Default connection to dev DB container { - "name": "Local Dev", + "name": "Local MariaDB", "server": "db", "port": 3306, "database": "ietf_utf8", "username": "django", "password": "RkTkDPFnKpko", - "driver": "MySQL", + "driver": "MariaDB", + "askForPassword": false, + "connectionTimeout": 60 + }, + { + "name": "Local PostgreSQL", + "server": "pgdb", + "port": 5432, + "database": "ietf", + "username": "django", + "password": "RkTkDPFnKpko", + "driver": "PostgreSQL", "askForPassword": false, "connectionTimeout": 60 } @@ -76,6 +87,7 @@ "ms-python.python", "ms-python.vscode-pylance", "mtxr.sqltools-driver-mysql", + "mtxr.sqltools-driver-pg", "mtxr.sqltools", "mutantdino.resourcemonitor", "oderwat.indent-rainbow", @@ -96,6 +108,10 @@ "label": "MariaDB", "onAutoForward": "silent" }, + "5432": { + "label": "PostgreSQL", + "onAutoForward": "silent" + }, "8000": { "label": "Datatracker", "onAutoForward": "notify" diff --git a/docker/db-pg.Dockerfile b/docker/db-pg.Dockerfile new file mode 100644 index 000000000..6f1792365 --- /dev/null +++ b/docker/db-pg.Dockerfile @@ -0,0 +1,8 @@ +FROM postgres:14.5 +LABEL maintainer="IETF Tools Team " + +ENV POSTGRES_PASSWORD=hk2j22sfiv +ENV POSTGRES_HOST_AUTH_METHOD=trust + +# Copy the postgres data folder from the migration stage +COPY ./pg-data /var/lib/postgresql/data diff --git a/docker/scripts/db-pg-migrate.sh b/docker/scripts/db-pg-migrate.sh new file mode 100644 index 000000000..57f547cc0 --- /dev/null +++ b/docker/scripts/db-pg-migrate.sh @@ -0,0 +1,66 @@ +export DEBIAN_FRONTEND=noninteractive + +chmod +x ./dev/tests/prepare.sh +sh ./dev/tests/prepare.sh + +mkdir -p pgdata + +# Setup pg database container +echo "Setting up PostgreSQL DB container..." +docker run -d --name pgdb -p 5432:5432 \ + -e POSTGRES_PASSWORD=RkTkDPFnKpko \ + -e POSTGRES_USER=django \ + -e POSTGRES_DB=ietf \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -v ./pgdata:/var/lib/postgresql/data + postgres:14.5 + +# Add Postgresql Apt Repository to get 14 +echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +# Install pg client and pgloader +apt-get update +apt-get install -y --no-install-recommends postgresql-client-14 pgloader + +# Copy settings files +cp ./docker/configs/settings_local.py ./ietf/settings_local.py +cp ./docker/configs/settings_mysqldb.py ./ietf/settings_mysqldb.py +cp ./docker/configs/settings_postgresqldb.py ./ietf/settings_postgresqldb.py + +# Wait for DB containers +echo "Waiting for DB containers to come online..." +/usr/local/bin/wait-for db:3306 -- echo "MariaDB ready" +/usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready" + +# Initial checks +echo "Running initial checks..." +/usr/local/bin/python ./ietf/manage.py check --settings=settings_local + +# Migrate, adjusting to what the current state of the underlying database might be: +/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local + +# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines) +/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local || true + +cat ./ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py + +# Now transfer the migrated database from mysql to postgres unless that's already happened. +echo "Transferring migrated database from MySQL to PostgreSQL..." +if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then + cat << EOF > cast.load +LOAD DATABASE +FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8 +INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf +CAST type varchar to text drop typemod; +EOF + time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load + rm cast.load + /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local +fi + +# Stop postgreSQL container +echo "Stopping PostgreSQL container..." +docker stop pgdb + +echo "Done."