chore: use prebuilt pg db image in dev (#4890)

2022-12-15 15:00:51 -05:00 · 2022-12-15 15:00:51 -05:00 · a60d52253e
parent 3d802497f4
commit a60d52253e
11 changed files with 21 additions and 128 deletions
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@ -47,20 +47,9 @@
        ],
        "sqltools.connections": [
            // Default connection to dev DB container
-            {
-                "name": "Local MariaDB",
-                "server": "db",
-                "port": 3306,
-                "database": "ietf_utf8",
-                "username": "django",
-                "password": "RkTkDPFnKpko",
-                "driver": "MariaDB",
-                "askForPassword": false,
-                "connectionTimeout": 60
-            },
            {
                "name": "Local PostgreSQL",
-                "server": "pgdb",
+                "server": "db",
                "port": 5432,
                "database": "ietf",
                "username": "django",
@ -86,7 +75,6 @@
        "ms-playwright.playwright",
 		"ms-python.python",
 		"ms-python.vscode-pylance",
-		"mtxr.sqltools-driver-mysql",
        "mtxr.sqltools-driver-pg",
        "mtxr.sqltools",
 		"mutantdino.resourcemonitor",
@ -97,17 +85,13 @@
 	],
    
    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [8000, 3306],
+    "forwardPorts": [8000, 5432],
    
    "portsAttributes": {
        "3000": {
            "label": "Vite",
            "onAutoForward": "silent"
        },
-        "3306": {
-            "label": "MariaDB",
-            "onAutoForward": "silent"
-        },
        "5432": {
            "label": "PostgreSQL",
            "onAutoForward": "silent"
--- a/dev/tests/docker-compose.debug.yml
+++ b/dev/tests/docker-compose.debug.yml
@ -17,17 +17,10 @@ services:
            GITHUB_ACTIONS: 'true'
            HOME: /github/home
    db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
+        image: ghcr.io/ietf-tools/datatracker-db-pg:latest
        restart: unless-stopped
        volumes:
-            - mariadb-data:/var/lib/mysql
-        environment:
-            MYSQL_ROOT_PASSWORD: RkTkDPFnKpko
-            MYSQL_DATABASE: ietf_utf8
-            MYSQL_USER: django
-            MYSQL_PASSWORD: RkTkDPFnKpko
-            CI: 'true'
-            GITHUB_ACTIONS: 'true'
+            - postgresdb-data:/var/lib/postgresql/data

 volumes:
-    mariadb-data:
+    postgresdb-data:
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@ -8,19 +8,14 @@ ALLOWED_HOSTS = ['*']
 DATABASES = {
    'default': {
        'HOST': 'db',
-        'PORT': 3306,
-        'NAME': 'ietf_utf8',
-        'ENGINE': 'django.db.backends.mysql',
+        'PORT': 5432,
+        'NAME': 'ietf',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'USER': 'django',
        'PASSWORD': 'RkTkDPFnKpko',
-        'OPTIONS': {
-            'sql_mode': 'STRICT_TRANS_TABLES',
-            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
-        },
    },
 }

-
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"
 IDSUBMIT_STAGING_PATH = "test/staging/"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -35,35 +35,11 @@ services:
        # (Adding the "ports" property to this file will not forward from a Codespace.)
                
    db:
-        image: ghcr.io/ietf-tools/datatracker-db:latest
+        image: ghcr.io/ietf-tools/datatracker-db-pg:latest
        #  build:
        #      context: ..
        #      dockerfile: docker/db.Dockerfile
        restart: unless-stopped
-        volumes:
-            - mariadb-data:/var/lib/mysql
-        environment:
-            MYSQL_ROOT_PASSWORD: RkTkDPFnKpko
-            MYSQL_DATABASE: ietf_utf8
-            MYSQL_USER: django
-            MYSQL_PASSWORD: RkTkDPFnKpko
-        command:
-            - '--character-set-server=utf8'
-            - '--collation-server=utf8_unicode_ci'
-            - '--innodb-buffer-pool-size=1G'
-            - '--innodb-log-buffer-size=128M'
-            - '--innodb-log-file-size=256M'
-            - '--innodb-write-io-threads=8'
-            - '--innodb-flush-log-at-trx-commit=0'
-            - '--performance-schema=1'
-
-    pgdb:
-        build:
-            context: .
-            dockerfile: docker/pgdb.Dockerfile
-        restart: unless-stopped
-        environment:
-            POSTGRES_PASSWORD: hk2j22sfiv
        volumes:
            - postgresdb-data:/var/lib/postgresql/data
            
@ -112,5 +88,4 @@ services:

 volumes:
    postgresdb-data:
-    mariadb-data:
    app-assets:
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@ -5,8 +5,7 @@ from ietf.settings import *                                          # pyflakes:

 ALLOWED_HOSTS = ['*']

-from ietf.settings_mysqldb import DATABASES
-
+from ietf.settings_postgresqldb import DATABASES

 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"
--- a/docker/configs/settings_postgresqldb.py
+++ b/docker/configs/settings_postgresqldb.py
@ -1,6 +1,6 @@
 DATABASES = {
    'default': {
-        'HOST': 'pgdb',
+        'HOST': 'db',
        'PORT': 5432,
        'NAME': 'ietf',
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
--- a/docker/docker-compose.extend.yml
+++ b/docker/docker-compose.extend.yml
@ -14,7 +14,7 @@ services:
            - app-assets:/assets
    db:
        ports:
-            - '3306'
+            - '5432'
    celery:
        volumes:
            - .:/workspace
--- a/docker/pgdb.Dockerfile
+++ b/docker/pgdb.Dockerfile
@ -1,19 +0,0 @@
-FROM postgres:14.5
-LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
-
-#RUN apt-get update \
-#    && apt-get install -y --no-install-recommends \
-#        postgresql-14-pg-catcheck \
-#        postgresql-14-powa \
-#        postgresql-14-pg-qualstats \
-#        postgresql-14-pg-stat-kcache \
-#        postgresql-14-pg-stat-monitor \
-#        postgresql-14-pg-top \
-#        postgresql-14-pg-track_settings \
-#        postgresql-14-pg-wait_sampling \
-#        pgsql_tweaks
-
-ENV POSTGRES_PASSWORD=hk2j22sfiv
-ENV POSTGRES_HOST_AUTH_METHOD=trust
-
-COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/
--- a/docker/scripts/app-init.sh
+++ b/docker/scripts/app-init.sh
@ -28,12 +28,8 @@ yarn build
 yarn legacy:build

 # Copy config files if needed
-if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then
-    cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
-fi
-if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then
-    cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py
-fi
+cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
+cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py

 if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then
    echo "Setting up a default settings_local.py ..."
@ -81,8 +77,8 @@ else
 fi

 # Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work)
-    python docker/scripts/db-include-fix.py
-    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+python docker/scripts/db-include-fix.py
+cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py

 # Create data directories

@ -99,10 +95,12 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/

 if [ -n "$EDITOR_VSCODE" ]; then
    echo "Waiting for DB container to come online ..."
-    /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready"
-    /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready"
+    /usr/local/bin/wait-for db:5432 -- echo "PostgreSQL ready"
 fi

+# Make sure PG search path is set
+psql -U django -h db -d ietf -v ON_ERROR_STOP=1 -c '\x' -c 'ALTER USER django set search_path=ietf_utf8,django,public;'
+
 # Run memcached

 echo "Starting memcached..."
@ -117,29 +115,6 @@ echo "Running initial checks..."

 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local

-
-# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines)
-/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
-
-echo "DEBUGGING pt 2 - this should say mysqldb"
-grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
-cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
-echo "DEBUGGING pt 3 - this should say postgresdb"
-grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
-
-# Now transfer the migrated database from mysql to postgres unless that's already happened.
-if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
-    cat << EOF > cast.load
-LOAD DATABASE
-FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
-INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf
-CAST type varchar to text drop typemod;
-EOF
-    time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
-    rm cast.load
-    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
-fi
-
 echo "-----------------------------------------------------------------"
 echo "Done!"
 echo "-----------------------------------------------------------------"
--- a/docker/scripts/db-include-fix.py
+++ b/docker/scripts/db-include-fix.py
@ -15,7 +15,7 @@ newcontent = content.replace(
        },
    },
 }""",
-    "from ietf.settings_mysqldb import DATABASES",
+    "from ietf.settings_postgresqldb import DATABASES",
 ).replace(
    """DATABASES = {
    'default': {
--- a/docker/scripts/pgdb-ietf-init.sh
+++ b/docker/scripts/pgdb-ietf-init.sh
@ -1,9 +0,0 @@
-#!/bin/bash
-set -e
-
-psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL
-	CREATE USER django PASSWORD 'RkTkDPFnKpko';
-	CREATE DATABASE ietf;
-	GRANT ALL PRIVILEGES ON DATABASE ietf TO django;
-    ALTER USER django set search_path=ietf_utf8,django,public;
-EOSQL