fix: simplify migrations when building postgres db image (#4889)

* chore: remove the effectively unused DATABASE_TEST_OPTIONS setting

* fix: simplify default settings_local so that db-include-fix.py is not needed in CI

* fix: simplify migrations in db-pg-migrate.sh

dev/deploy-to-container/settings_local.py

@@ -20,10 +20,6 @@ DATABASES = {
     },
 }
 
-DATABASE_TEST_OPTIONS = {
-    'init_command': 'SET storage_engine=InnoDB',
-}
-
 SECRET_KEY = "__SECRETKEY__"
 
 CELERY_BROKER_URL = '__MQCONNSTR__'

dev/diff/settings_local.py

@@ -20,9 +20,6 @@ DATABASES = {
     },
 }
 
-DATABASE_TEST_OPTIONS = {
-    'init_command': 'SET storage_engine=InnoDB',
-}
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"

dev/tests/settings_local.py

@@ -20,9 +20,6 @@ DATABASES = {
     },
 }
 
-DATABASE_TEST_OPTIONS = {
-    'init_command': 'SET storage_engine=InnoDB',
-}
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"

docker/configs/settings_local.py

@@ -5,24 +5,8 @@ from ietf.settings import *                                          # pyflakes:
 
 ALLOWED_HOSTS = ['*']
 
-DATABASES = {
+from ietf.settings_mysqldb import DATABASES
-    'default': {
-        'HOST': 'db',
-        'PORT': 3306,
-        'NAME': 'ietf_utf8',
-        'ENGINE': 'django.db.backends.mysql',
-        'USER': 'django',
-        'PASSWORD': 'RkTkDPFnKpko',
-        'OPTIONS': {
-            'sql_mode': 'STRICT_TRANS_TABLES',
-            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
-        },
-    },
-}
 
-DATABASE_TEST_OPTIONS = {
-    'init_command': 'SET storage_engine=InnoDB',
-}
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
 IDSUBMIT_REPOSITORY_PATH = "test/id/"

docker/configs/settings_mysqldb.py

@@ -12,7 +12,3 @@ DATABASES = {
         },
     },
 }
-
-DATABASE_TEST_OPTIONS = {
-    'init_command': 'SET storage_engine=InnoDB',
-}

docker/scripts/db-pg-migrate.sh

@@ -45,17 +45,21 @@ echo "Waiting for DB containers to come online..."
 echo "Running initial checks..."
 /usr/local/bin/python ./ietf/manage.py check --settings=settings_local
 
-# Migrate, adjusting to what the current state of the underlying database might be:
+# The mysql database is always freshly build container from the 
-/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
+# image build of last-night's dump when this script is run
-
+# The first run of migrations will run anything merged from main that
-# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines)
+# that hasn't been released, and the few pre-engine-shift migrations
+# that the feat/postgres branch adds. It is guaranteed to fail at
+# utils.migrations.0004_pause_to_change_database_engines (where it
+# fails on purpose, hence the `|| true` so we may proceed
 /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local || true
 
 cat ./ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
 
 # Now transfer the migrated database from mysql to postgres unless that's already happened.
 echo "Transferring migrated database from MySQL to PostgreSQL..."
-if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
+EMPTY_CHECK=`psql -U django -h pgdb -d ietf -c "\dt" 2>&1`
+if echo ${EMPTY_CHECK} | grep -q "Did not find any relations."; then
     cat << EOF > cast.load
 LOAD DATABASE
 FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
@@ -65,6 +69,9 @@ EOF
     time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
     rm cast.load
     /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
+else
+    echo "The postgres database is in an unexpected state"
+    echo ${EMPTY_CHECK}
 fi
 
 # Stop postgreSQL container

ietf/settings.py

@@ -91,10 +91,6 @@ DATABASES = {
     },
 }
 
-DATABASE_TEST_OPTIONS = {
-    # Comment this out if your database doesn't support InnoDB
-    'init_command': 'SET storage_engine=InnoDB',
-}
 
 # Local time zone for this installation. Choices can be found here:
 # http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE

ietf/utils/test_runner.py

@@ -229,9 +229,6 @@ def safe_create_test_db(self, verbosity, *args, **kwargs):
     keepdb = kwargs.get('keepdb', False)
     if not keepdb:
         print("     Creating test database...")
-        if settings.DATABASES["default"]["ENGINE"] == 'django.db.backends.mysql':
-            settings.DATABASES["default"]["OPTIONS"] = settings.DATABASE_TEST_OPTIONS
-            print("     Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"])
     test_database_name = old_create(self, 0, *args, **kwargs)
 
     if settings.GLOBAL_TEST_FIXTURES: