chore: use prebuilt pg db image in dev (#4890)

.devcontainer/devcontainer.json

@@ -47,20 +47,9 @@
         ],
         "sqltools.connections": [
             // Default connection to dev DB container
-            {
-                "name": "Local MariaDB",
-                "server": "db",
-                "port": 3306,
-                "database": "ietf_utf8",
-                "username": "django",
-                "password": "RkTkDPFnKpko",
-                "driver": "MariaDB",
-                "askForPassword": false,
-                "connectionTimeout": 60
-            },
             {
                 "name": "Local PostgreSQL",
-                "server": "pgdb",
+                "server": "db",
                 "port": 5432,
                 "database": "ietf",
                 "username": "django",
@@ -86,7 +75,6 @@
         "ms-playwright.playwright",
 		"ms-python.python",
 		"ms-python.vscode-pylance",
-		"mtxr.sqltools-driver-mysql",
         "mtxr.sqltools-driver-pg",
         "mtxr.sqltools",
 		"mutantdino.resourcemonitor",
@@ -97,17 +85,13 @@
 	],
     
     // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [8000, 3306],
+    "forwardPorts": [8000, 5432],
     
     "portsAttributes": {
         "3000": {
             "label": "Vite",
             "onAutoForward": "silent"
         },
-        "3306": {
-            "label": "MariaDB",
-            "onAutoForward": "silent"
-        },
         "5432": {
             "label": "PostgreSQL",
             "onAutoForward": "silent"

dev/tests/docker-compose.debug.yml

@@ -17,17 +17,10 @@ services:
             GITHUB_ACTIONS: 'true'
             HOME: /github/home
     db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
+        image: ghcr.io/ietf-tools/datatracker-db-pg:latest
         restart: unless-stopped
         volumes:
-            - mariadb-data:/var/lib/mysql
+            - postgresdb-data:/var/lib/postgresql/data
-        environment:
-            MYSQL_ROOT_PASSWORD: RkTkDPFnKpko
-            MYSQL_DATABASE: ietf_utf8
-            MYSQL_USER: django
-            MYSQL_PASSWORD: RkTkDPFnKpko
-            CI: 'true'
-            GITHUB_ACTIONS: 'true'
 
 volumes:
-    mariadb-data:
+    postgresdb-data:

dev/tests/settings_local.py

@@ -8,19 +8,14 @@ ALLOWED_HOSTS = ['*']
 DATABASES = {
     'default': {
         'HOST': 'db',
-        'PORT': 3306,
+        'PORT': 5432,
-        'NAME': 'ietf_utf8',
+        'NAME': 'ietf',
-        'ENGINE': 'django.db.backends.mysql',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
         'USER': 'django',
         'PASSWORD': 'RkTkDPFnKpko',
-        'OPTIONS': {
-            'sql_mode': 'STRICT_TRANS_TABLES',
-            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
-        },
     },
 }
 
-
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"
 IDSUBMIT_STAGING_PATH = "test/staging/"

docker-compose.yml

@@ -35,35 +35,11 @@ services:
         # (Adding the "ports" property to this file will not forward from a Codespace.)
                 
     db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
+        image: ghcr.io/ietf-tools/datatracker-db-pg:latest
         #  build:
         #      context: ..
         #      dockerfile: docker/db.Dockerfile
         restart: unless-stopped
-        volumes:
-            - mariadb-data:/var/lib/mysql
-        environment:
-            MYSQL_ROOT_PASSWORD: RkTkDPFnKpko
-            MYSQL_DATABASE: ietf_utf8
-            MYSQL_USER: django
-            MYSQL_PASSWORD: RkTkDPFnKpko
-        command:
-            - '--character-set-server=utf8'
-            - '--collation-server=utf8_unicode_ci'
-            - '--innodb-buffer-pool-size=1G'
-            - '--innodb-log-buffer-size=128M'
-            - '--innodb-log-file-size=256M'
-            - '--innodb-write-io-threads=8'
-            - '--innodb-flush-log-at-trx-commit=0'
-            - '--performance-schema=1'
-
-    pgdb:
-        build:
-            context: .
-            dockerfile: docker/pgdb.Dockerfile
-        restart: unless-stopped
-        environment:
-            POSTGRES_PASSWORD: hk2j22sfiv
         volumes:
             - postgresdb-data:/var/lib/postgresql/data
             
@@ -112,5 +88,4 @@ services:
 
 volumes:
     postgresdb-data:
-    mariadb-data:
     app-assets:

docker/configs/settings_local.py

@@ -5,8 +5,7 @@ from ietf.settings import *                                          # pyflakes:
 
 ALLOWED_HOSTS = ['*']
 
-from ietf.settings_mysqldb import DATABASES
+from ietf.settings_postgresqldb import DATABASES
-
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"

docker/configs/settings_postgresqldb.py

@@ -1,6 +1,6 @@
 DATABASES = {
     'default': {
-        'HOST': 'pgdb',
+        'HOST': 'db',
         'PORT': 5432,
         'NAME': 'ietf',
         'ENGINE': 'django.db.backends.postgresql_psycopg2',

docker/docker-compose.extend.yml

@@ -14,7 +14,7 @@ services:
             - app-assets:/assets
     db:
         ports:
-            - '3306'
+            - '5432'
     celery:
         volumes:
             - .:/workspace

docker/pgdb.Dockerfile

@@ -1,19 +0,0 @@
-FROM postgres:14.5
-LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
-
-#RUN apt-get update \
-#    && apt-get install -y --no-install-recommends \
-#        postgresql-14-pg-catcheck \
-#        postgresql-14-powa \
-#        postgresql-14-pg-qualstats \
-#        postgresql-14-pg-stat-kcache \
-#        postgresql-14-pg-stat-monitor \
-#        postgresql-14-pg-top \
-#        postgresql-14-pg-track_settings \
-#        postgresql-14-pg-wait_sampling \
-#        pgsql_tweaks
-
-ENV POSTGRES_PASSWORD=hk2j22sfiv
-ENV POSTGRES_HOST_AUTH_METHOD=trust
-
-COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/

docker/scripts/app-init.sh

@@ -28,12 +28,8 @@ yarn build
 yarn legacy:build
 
 # Copy config files if needed
-if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then
+cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
-    cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
+cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py
-fi
-if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then
-    cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py
-fi
 
 if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then
     echo "Setting up a default settings_local.py ..."
@@ -81,8 +77,8 @@ else
 fi
 
 # Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work)
-    python docker/scripts/db-include-fix.py
+python docker/scripts/db-include-fix.py
-    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
 
 # Create data directories
 
@@ -99,10 +95,12 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/
 
 if [ -n "$EDITOR_VSCODE" ]; then
     echo "Waiting for DB container to come online ..."
-    /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready"
+    /usr/local/bin/wait-for db:5432 -- echo "PostgreSQL ready"
-    /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready"
 fi
 
+# Make sure PG search path is set
+psql -U django -h db -d ietf -v ON_ERROR_STOP=1 -c '\x' -c 'ALTER USER django set search_path=ietf_utf8,django,public;'
+
 # Run memcached
 
 echo "Starting memcached..."
@@ -117,29 +115,6 @@ echo "Running initial checks..."
 
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
 
-
-# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines)
-/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
-
-echo "DEBUGGING pt 2 - this should say mysqldb"
-grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
-cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
-echo "DEBUGGING pt 3 - this should say postgresdb"
-grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
-
-# Now transfer the migrated database from mysql to postgres unless that's already happened.
-if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
-    cat << EOF > cast.load
-LOAD DATABASE
-FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
-INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf
-CAST type varchar to text drop typemod;
-EOF
-    time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
-    rm cast.load
-    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
-fi
-
 echo "-----------------------------------------------------------------"
 echo "Done!"
 echo "-----------------------------------------------------------------"

docker/scripts/db-include-fix.py

@@ -15,7 +15,7 @@ newcontent = content.replace(
         },
     },
 }""",
-    "from ietf.settings_mysqldb import DATABASES",
+    "from ietf.settings_postgresqldb import DATABASES",
 ).replace(
     """DATABASES = {
     'default': {

docker/scripts/pgdb-ietf-init.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL
-	CREATE USER django PASSWORD 'RkTkDPFnKpko';
-	CREATE DATABASE ietf;
-	GRANT ALL PRIVILEGES ON DATABASE ietf TO django;
-    ALTER USER django set search_path=ietf_utf8,django,public;
-EOSQL