feat: move to postgresql (#4744)

* feat: move to postgresql

* fix: repair fractional replace statement

* fix: use pathlib to manipulate settings_local

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

* fix: do two string replacements, not one followed by another that throws away the first.

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

* fix: use pathlib again to manipulate settings_local

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

* fix: properly use assert (1/2)

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

* fix: properly use assert (2/2)

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

Co-authored-by: Jennifer Richards <jennifer@painless-security.com>

docker-compose.yml

@@ -56,6 +56,16 @@ services:
             - '--innodb-write-io-threads=8'
             - '--innodb-flush-log-at-trx-commit=0'
             - '--performance-schema=1'
+
+    pgdb:
+        build:
+            context: .
+            dockerfile: docker/pgdb.Dockerfile
+        restart: unless-stopped
+        environment:
+            POSTGRES_PASSWORD: hk2j22sfiv
+        volumes:
+            - postgresdb-data:/var/lib/postgresql/data
             
     # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
     # (Adding the "ports" property to this file will not forward from a Codespace.)
@@ -101,5 +111,6 @@ services:
 #            - .:/workspace
 
 volumes:
+    postgresdb-data:
     mariadb-data:
     app-assets:

docker/app.Dockerfile

@@ -9,7 +9,13 @@ ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh
 RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh
+
+# Add Postgresql Apt Repository to get 14    
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
     && apt-get purge -y imagemagick imagemagick-6-common \
     # Install common packages, non-root user

docker/configs/settings_mysqldb.py

@@ -0,0 +1,18 @@
+DATABASES = {
+    'default': {
+        'HOST': 'db',
+        'PORT': 3306,
+        'NAME': 'ietf_utf8',
+        'ENGINE': 'django.db.backends.mysql',
+        'USER': 'django',
+        'PASSWORD': 'RkTkDPFnKpko',
+        'OPTIONS': {
+            'sql_mode': 'STRICT_TRANS_TABLES',
+            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
+        },
+    },
+}
+
+DATABASE_TEST_OPTIONS = {
+    'init_command': 'SET storage_engine=InnoDB',
+}

docker/configs/settings_postgresqldb.py

@@ -0,0 +1,10 @@
+DATABASES = {
+    'default': {
+        'HOST': 'pgdb',
+        'PORT': 5432,
+        'NAME': 'ietf',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'USER': 'django',
+        'PASSWORD': 'RkTkDPFnKpko',
+    },
+}

docker/pgdb.Dockerfile

@@ -0,0 +1,19 @@
+FROM postgres:14.5
+LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"
+
+#RUN apt-get update \
+#    && apt-get install -y --no-install-recommends \
+#        postgresql-14-pg-catcheck \
+#        postgresql-14-powa \
+#        postgresql-14-pg-qualstats \
+#        postgresql-14-pg-stat-kcache \
+#        postgresql-14-pg-stat-monitor \
+#        postgresql-14-pg-top \
+#        postgresql-14-pg-track_settings \
+#        postgresql-14-pg-wait_sampling \
+#        pgsql_tweaks
+
+ENV POSTGRES_PASSWORD=hk2j22sfiv
+ENV POSTGRES_HOST_AUTH_METHOD=trust
+
+COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/

docker/scripts/app-init.sh

@@ -28,10 +28,17 @@ yarn build
 yarn legacy:build
 
 # Copy config files if needed
+if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then
+    cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py
+fi
+if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then
+    cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py
+fi
 
 if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then
     echo "Setting up a default settings_local.py ..."
     cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+
 else
     echo "Using existing ietf/settings_local.py file"
     if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then
@@ -73,6 +80,10 @@ else
     fi
 fi
 
+# Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work)
+    python docker/scripts/db-include-fix.py
+    cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+
 # Create data directories
 
 echo "Creating data directories..."
@@ -88,7 +99,8 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/
 
 if [ -n "$EDITOR_VSCODE" ]; then
     echo "Waiting for DB container to come online ..."
-    /usr/local/bin/wait-for localhost:3306 -- echo "DB ready"
+    /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready"
+    /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready"
 fi
 
 # Run memcached
@@ -112,7 +124,11 @@ if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; the
     # This is expected to exit non-zero at the pause
     /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
     cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
-    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+    # This is also expected to exit non-zero at the 2nd pause
+    echo "DEBUGGING pt 1 - this should say mysqldb"
+    grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true
+    # More migrations after the move to postgres below.
 
 else
     if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then
@@ -123,6 +139,25 @@ else
     fi
 fi
 
+echo "DEBUGGING pt 2 - this should say mysqldb"
+grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
+cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
+echo "DEBUGGING pt 3 - this should say postgresdb"
+grep "DATA" $WORKSPACEDIR/ietf/settings_local.py
+
+# Now transfer the migrated database from mysql to postgres unless that's already happened.
+if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
+    cat << EOF > cast.load
+LOAD DATABASE
+FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
+INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf
+CAST type varchar to text drop typemod;
+EOF
+    time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
+    rm cast.load
+    /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local
+fi
+
 echo "-----------------------------------------------------------------"
 echo "Done!"
 echo "-----------------------------------------------------------------"

docker/scripts/db-include-fix.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+content = Path('/workspace/ietf/settings_local.py').read_text()
+newcontent = content.replace(
+    """DATABASES = {
+    'default': {
+        'HOST': 'db',
+        'PORT': 3306,
+        'NAME': 'ietf_utf8',
+        'ENGINE': 'django.db.backends.mysql',
+        'USER': 'django',
+        'PASSWORD': 'RkTkDPFnKpko',
+        'OPTIONS': {
+            'sql_mode': 'STRICT_TRANS_TABLES',
+            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
+        },
+    },
+}""",
+    "from ietf.settings_mysqldb import DATABASES",
+).replace(
+    """DATABASES = {
+    'default': {
+        'HOST': 'pgdb',
+        'PORT': 5432,
+        'NAME': 'ietf',
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'USER': 'django',
+        'PASSWORD': 'RkTkDPFnKpko',
+    },
+}""",
+    "from ietf.settings_postgresqldb import DATABASES",
+)
+with Path('/workspace/ietf/settings_local.py').open('w') as replacementfile:
+    replacementfile.write(newcontent)

docker/scripts/pgdb-ietf-init.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL
+	CREATE USER django PASSWORD 'RkTkDPFnKpko';
+	CREATE DATABASE ietf;
+	GRANT ALL PRIVILEGES ON DATABASE ietf TO django;
+    ALTER USER django set search_path=ietf_utf8,django,public;
+EOSQL

ietf/.gitignore

@@ -4,4 +4,6 @@
 /settings_local_debug.py
 /settings_local_sqlitetest.py
 /settings_local_vite.py
+/settings_mysqldb.py
+/settings_postgresqldb.py
 /ietfdb.sql.gz

ietf/community/migrations/0010_doc_ids_are_ints.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.28 on 2022-11-05 11:07
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('community', '0009_add_group_exp_rule_to_groups'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql="alter table community_communitylist_added_docs modify document_id int unsigned not null;",
+            reverse_sql="alter table community_communitylist_added_docs modify document_id varchar(255) not null;"
+        ),
+                migrations.RunSQL(
+            sql="alter table community_searchrule_name_contains_index modify document_id int unsigned not null;",
+            reverse_sql="alter table community_searchrule_name_contains_index modify document_id varchar(255) not null;"
+        ),
+    ]

ietf/meeting/migrations/0059_durationfield_are_intervals.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.28 on 2022-11-10 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('meeting', '0058_meeting_time_zone_not_blank'),
+        ('utils', '0004_pause_to_change_database_engines'),
+    ]
+
+    operations = [
+        # reversesql is intentionally not provided. There is no looking back...
+        migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_time_utc=idsubmit_cutoff_time_utc/1000"),
+        migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_time_utc type interval using (idsubmit_cutoff_time_utc::text||' milliseconds')::interval;"),
+        migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_warning_days=idsubmit_cutoff_warning_days/1000"),
+        migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_warning_days type interval using (idsubmit_cutoff_warning_days::text||' milliseconds')::interval;"),
+        migrations.RunSQL(sql="update meeting_timeslot set duration=duration/1000"),
+        migrations.RunSQL(sql="alter table meeting_timeslot alter column duration type interval using (duration::text||' milliseconds')::interval;"),
+        migrations.RunSQL(sql="update meeting_session set requested_duration=requested_duration/1000"),
+        migrations.RunSQL(sql="alter table meeting_session alter column requested_duration type interval using (requested_duration::text||' milliseconds')::interval;"),      
+    ]

ietf/utils/migrations/0004_pause_to_change_database_engines.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.2.28 on 2022-11-10 09:27
+
+from django.conf import settings
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing'
+
+def reverse(apps, schema_editor):
+    assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing'
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('utils', '0003_pause_to_change_use_tz'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

requirements.txt

@@ -45,6 +45,7 @@ mypy>=0.782,<0.790    # Version requirements determined by django-stubs.
 mysqlclient>=2.1.0
 oic>=1.3    # Used only by tests
 Pillow>=9.1.0
+psycopg2<2.9
 pyang>=2.5.3
 pyflakes>=2.4.0
 pyopenssl>=22.0.0    # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency