From 27fccc6ba05cec8d581b0d7afe3a1d7a982916e3 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 11 Nov 2022 11:01:01 +0000 Subject: [PATCH] feat: move to postgresql (#4744) * feat: move to postgresql * fix: repair fractional replace statement * fix: use pathlib to manipulate settings_local Co-authored-by: Jennifer Richards * fix: do two string replacements, not one followed by another that throws away the first. Co-authored-by: Jennifer Richards * fix: use pathlib again to manipulate settings_local Co-authored-by: Jennifer Richards * fix: properly use assert (1/2) Co-authored-by: Jennifer Richards * fix: properly use assert (2/2) Co-authored-by: Jennifer Richards Co-authored-by: Jennifer Richards --- docker-compose.yml | 11 ++++++ docker/app.Dockerfile | 6 +++ docker/configs/settings_mysqldb.py | 18 +++++++++ docker/configs/settings_postgresqldb.py | 10 +++++ docker/pgdb.Dockerfile | 19 +++++++++ docker/scripts/app-init.sh | 39 ++++++++++++++++++- docker/scripts/db-include-fix.py | 33 ++++++++++++++++ docker/scripts/pgdb-ietf-init.sh | 9 +++++ ietf/.gitignore | 2 + .../migrations/0010_doc_ids_are_ints.py | 21 ++++++++++ .../0059_durationfield_are_intervals.py | 23 +++++++++++ .../0004_pause_to_change_database_engines.py | 20 ++++++++++ requirements.txt | 1 + 13 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 docker/configs/settings_mysqldb.py create mode 100644 docker/configs/settings_postgresqldb.py create mode 100644 docker/pgdb.Dockerfile create mode 100644 docker/scripts/db-include-fix.py create mode 100755 docker/scripts/pgdb-ietf-init.sh create mode 100644 ietf/community/migrations/0010_doc_ids_are_ints.py create mode 100644 ietf/meeting/migrations/0059_durationfield_are_intervals.py create mode 100644 ietf/utils/migrations/0004_pause_to_change_database_engines.py diff --git a/docker-compose.yml b/docker-compose.yml index db8ea9152..b25a64ef9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,16 @@ services: - '--innodb-write-io-threads=8' - '--innodb-flush-log-at-trx-commit=0' - '--performance-schema=1' + + pgdb: + build: + context: . + dockerfile: docker/pgdb.Dockerfile + restart: unless-stopped + environment: + POSTGRES_PASSWORD: hk2j22sfiv + volumes: + - postgresdb-data:/var/lib/postgresql/data # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) @@ -101,5 +111,6 @@ services: # - .:/workspace volumes: + postgresdb-data: mariadb-data: app-assets: diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 6d539b1a5..af475a4e6 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -9,7 +9,13 @@ ARG USER_UID=1000 ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh + +# Add Postgresql Apt Repository to get 14 +RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ + && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/configs/settings_mysqldb.py b/docker/configs/settings_mysqldb.py new file mode 100644 index 000000000..445f01e70 --- /dev/null +++ b/docker/configs/settings_mysqldb.py @@ -0,0 +1,18 @@ +DATABASES = { + 'default': { + 'HOST': 'db', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +} + +DATABASE_TEST_OPTIONS = { + 'init_command': 'SET storage_engine=InnoDB', +} diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py new file mode 100644 index 000000000..e400420b5 --- /dev/null +++ b/docker/configs/settings_postgresqldb.py @@ -0,0 +1,10 @@ +DATABASES = { + 'default': { + 'HOST': 'pgdb', + 'PORT': 5432, + 'NAME': 'ietf', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + }, +} diff --git a/docker/pgdb.Dockerfile b/docker/pgdb.Dockerfile new file mode 100644 index 000000000..e566ab645 --- /dev/null +++ b/docker/pgdb.Dockerfile @@ -0,0 +1,19 @@ +FROM postgres:14.5 +LABEL maintainer="IETF Tools Team " + +#RUN apt-get update \ +# && apt-get install -y --no-install-recommends \ +# postgresql-14-pg-catcheck \ +# postgresql-14-powa \ +# postgresql-14-pg-qualstats \ +# postgresql-14-pg-stat-kcache \ +# postgresql-14-pg-stat-monitor \ +# postgresql-14-pg-top \ +# postgresql-14-pg-track_settings \ +# postgresql-14-pg-wait_sampling \ +# pgsql_tweaks + +ENV POSTGRES_PASSWORD=hk2j22sfiv +ENV POSTGRES_HOST_AUTH_METHOD=trust + +COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/ diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 7557e8f45..94a123236 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -28,10 +28,17 @@ yarn build yarn legacy:build # Copy config files if needed +if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then + cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py +fi +if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then + cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py +fi if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then echo "Setting up a default settings_local.py ..." cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py + else echo "Using existing ietf/settings_local.py file" if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then @@ -73,6 +80,10 @@ else fi fi +# Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work) + python docker/scripts/db-include-fix.py + cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py + # Create data directories echo "Creating data directories..." @@ -88,7 +99,8 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/ if [ -n "$EDITOR_VSCODE" ]; then echo "Waiting for DB container to come online ..." - /usr/local/bin/wait-for localhost:3306 -- echo "DB ready" + /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready" + /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready" fi # Run memcached @@ -112,7 +124,11 @@ if ietf/manage.py showmigrations | grep "\[ \] 0003_pause_to_change_use_tz"; the # This is expected to exit non-zero at the pause /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/USE_TZ.*$/USE_TZ = True/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py - /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local + # This is also expected to exit non-zero at the 2nd pause + echo "DEBUGGING pt 1 - this should say mysqldb" + grep "DATA" $WORKSPACEDIR/ietf/settings_local.py + /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true + # More migrations after the move to postgres below. else if grep "USE_TZ" $WORKSPACEDIR/ietf/settings_local.py; then @@ -123,6 +139,25 @@ else fi fi +echo "DEBUGGING pt 2 - this should say mysqldb" +grep "DATA" $WORKSPACEDIR/ietf/settings_local.py +cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py +echo "DEBUGGING pt 3 - this should say postgresdb" +grep "DATA" $WORKSPACEDIR/ietf/settings_local.py + +# Now transfer the migrated database from mysql to postgres unless that's already happened. +if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then + cat << EOF > cast.load +LOAD DATABASE +FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8 +INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf +CAST type varchar to text drop typemod; +EOF + time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load + rm cast.load + /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local +fi + echo "-----------------------------------------------------------------" echo "Done!" echo "-----------------------------------------------------------------" diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py new file mode 100644 index 000000000..369867061 --- /dev/null +++ b/docker/scripts/db-include-fix.py @@ -0,0 +1,33 @@ +from pathlib import Path +content = Path('/workspace/ietf/settings_local.py').read_text() +newcontent = content.replace( + """DATABASES = { + 'default': { + 'HOST': 'db', + 'PORT': 3306, + 'NAME': 'ietf_utf8', + 'ENGINE': 'django.db.backends.mysql', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + 'OPTIONS': { + 'sql_mode': 'STRICT_TRANS_TABLES', + 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', + }, + }, +}""", + "from ietf.settings_mysqldb import DATABASES", +).replace( + """DATABASES = { + 'default': { + 'HOST': 'pgdb', + 'PORT': 5432, + 'NAME': 'ietf', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'USER': 'django', + 'PASSWORD': 'RkTkDPFnKpko', + }, +}""", + "from ietf.settings_postgresqldb import DATABASES", +) +with Path('/workspace/ietf/settings_local.py').open('w') as replacementfile: + replacementfile.write(newcontent) diff --git a/docker/scripts/pgdb-ietf-init.sh b/docker/scripts/pgdb-ietf-init.sh new file mode 100755 index 000000000..533084056 --- /dev/null +++ b/docker/scripts/pgdb-ietf-init.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL + CREATE USER django PASSWORD 'RkTkDPFnKpko'; + CREATE DATABASE ietf; + GRANT ALL PRIVILEGES ON DATABASE ietf TO django; + ALTER USER django set search_path=ietf_utf8,django,public; +EOSQL diff --git a/ietf/.gitignore b/ietf/.gitignore index 322cc214d..9df45d76a 100644 --- a/ietf/.gitignore +++ b/ietf/.gitignore @@ -4,4 +4,6 @@ /settings_local_debug.py /settings_local_sqlitetest.py /settings_local_vite.py +/settings_mysqldb.py +/settings_postgresqldb.py /ietfdb.sql.gz diff --git a/ietf/community/migrations/0010_doc_ids_are_ints.py b/ietf/community/migrations/0010_doc_ids_are_ints.py new file mode 100644 index 000000000..a76eda92a --- /dev/null +++ b/ietf/community/migrations/0010_doc_ids_are_ints.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.28 on 2022-11-05 11:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('community', '0009_add_group_exp_rule_to_groups'), + ] + + operations = [ + migrations.RunSQL( + sql="alter table community_communitylist_added_docs modify document_id int unsigned not null;", + reverse_sql="alter table community_communitylist_added_docs modify document_id varchar(255) not null;" + ), + migrations.RunSQL( + sql="alter table community_searchrule_name_contains_index modify document_id int unsigned not null;", + reverse_sql="alter table community_searchrule_name_contains_index modify document_id varchar(255) not null;" + ), + ] diff --git a/ietf/meeting/migrations/0059_durationfield_are_intervals.py b/ietf/meeting/migrations/0059_durationfield_are_intervals.py new file mode 100644 index 000000000..b30b5acea --- /dev/null +++ b/ietf/meeting/migrations/0059_durationfield_are_intervals.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.28 on 2022-11-10 02:35 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('meeting', '0058_meeting_time_zone_not_blank'), + ('utils', '0004_pause_to_change_database_engines'), + ] + + operations = [ + # reversesql is intentionally not provided. There is no looking back... + migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_time_utc=idsubmit_cutoff_time_utc/1000"), + migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_time_utc type interval using (idsubmit_cutoff_time_utc::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_meeting set idsubmit_cutoff_warning_days=idsubmit_cutoff_warning_days/1000"), + migrations.RunSQL(sql="alter table meeting_meeting alter column idsubmit_cutoff_warning_days type interval using (idsubmit_cutoff_warning_days::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_timeslot set duration=duration/1000"), + migrations.RunSQL(sql="alter table meeting_timeslot alter column duration type interval using (duration::text||' milliseconds')::interval;"), + migrations.RunSQL(sql="update meeting_session set requested_duration=requested_duration/1000"), + migrations.RunSQL(sql="alter table meeting_session alter column requested_duration type interval using (requested_duration::text||' milliseconds')::interval;"), + ] diff --git a/ietf/utils/migrations/0004_pause_to_change_database_engines.py b/ietf/utils/migrations/0004_pause_to_change_database_engines.py new file mode 100644 index 000000000..58a502b25 --- /dev/null +++ b/ietf/utils/migrations/0004_pause_to_change_database_engines.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.28 on 2022-11-10 09:27 + +from django.conf import settings +from django.db import migrations + +def forward(apps, schema_editor): + assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2', 'Please change to use postgres before continuing' + +def reverse(apps, schema_editor): + assert settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql', 'Please change to use mariadb before continuing' + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0003_pause_to_change_use_tz'), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/requirements.txt b/requirements.txt index 155e5c676..dd5349a7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,6 +45,7 @@ mypy>=0.782,<0.790 # Version requirements determined by django-stubs. mysqlclient>=2.1.0 oic>=1.3 # Used only by tests Pillow>=9.1.0 +psycopg2<2.9 pyang>=2.5.3 pyflakes>=2.4.0 pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency