From 912c7b5ee3777804cd958f6c08fc77c13cc6793d Mon Sep 17 00:00:00 2001
From: Robert Sparks <rjsparks@nostrum.com>
Date: Tue, 13 Dec 2022 16:37:24 -0600
Subject: [PATCH] fix: simplify migrations when building postgres db image
(#4889)
* chore: remove the effectively unused DATABASE_TEST_OPTIONS setting
* fix: simplify default settings_local so that db-include-fix.py is not needed in CI
* fix: simplify migrations in db-pg-migrate.sh
---
dev/deploy-to-container/settings_local.py | 4 ----
dev/diff/settings_local.py | 3 ---
dev/tests/settings_local.py | 3 ---
docker/configs/settings_local.py | 18 +-----------------
docker/configs/settings_mysqldb.py | 4 ----
docker/scripts/db-pg-migrate.sh | 17 ++++++++++++-----
ietf/settings.py | 4 ----
ietf/utils/test_runner.py | 3 ---
8 files changed, 13 insertions(+), 43 deletions(-)
diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py
index f4753cc8a..f1b86205b 100644
--- a/dev/deploy-to-container/settings_local.py
+++ b/dev/deploy-to-container/settings_local.py
@@ -20,10 +20,6 @@ DATABASES = {
},
}
-DATABASE_TEST_OPTIONS = {
- 'init_command': 'SET storage_engine=InnoDB',
-}
-
SECRET_KEY = "__SECRETKEY__"
CELERY_BROKER_URL = '__MQCONNSTR__'
diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py
index 902fc54d1..0a89f5c8e 100644
--- a/dev/diff/settings_local.py
+++ b/dev/diff/settings_local.py
@@ -20,9 +20,6 @@ DATABASES = {
},
}
-DATABASE_TEST_OPTIONS = {
- 'init_command': 'SET storage_engine=InnoDB',
-}
IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
IDSUBMIT_REPOSITORY_PATH = "test/id/"
diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py
index 2b3d54138..16f36b6d8 100644
--- a/dev/tests/settings_local.py
+++ b/dev/tests/settings_local.py
@@ -20,9 +20,6 @@ DATABASES = {
},
}
-DATABASE_TEST_OPTIONS = {
- 'init_command': 'SET storage_engine=InnoDB',
-}
IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
IDSUBMIT_REPOSITORY_PATH = "test/id/"
diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py
index afcb5b202..cee35097e 100644
--- a/docker/configs/settings_local.py
+++ b/docker/configs/settings_local.py
@@ -5,24 +5,8 @@ from ietf.settings import * # pyflakes:
ALLOWED_HOSTS = ['*']
-DATABASES = {
- 'default': {
- 'HOST': 'db',
- 'PORT': 3306,
- 'NAME': 'ietf_utf8',
- 'ENGINE': 'django.db.backends.mysql',
- 'USER': 'django',
- 'PASSWORD': 'RkTkDPFnKpko',
- 'OPTIONS': {
- 'sql_mode': 'STRICT_TRANS_TABLES',
- 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
- },
- },
-}
+from ietf.settings_mysqldb import DATABASES
-DATABASE_TEST_OPTIONS = {
- 'init_command': 'SET storage_engine=InnoDB',
-}
IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
IDSUBMIT_REPOSITORY_PATH = "test/id/"
diff --git a/docker/configs/settings_mysqldb.py b/docker/configs/settings_mysqldb.py
index 445f01e70..ee2336846 100644
--- a/docker/configs/settings_mysqldb.py
+++ b/docker/configs/settings_mysqldb.py
@@ -12,7 +12,3 @@ DATABASES = {
},
},
}
-
-DATABASE_TEST_OPTIONS = {
- 'init_command': 'SET storage_engine=InnoDB',
-}
diff --git a/docker/scripts/db-pg-migrate.sh b/docker/scripts/db-pg-migrate.sh
index 29507f237..480b64209 100644
--- a/docker/scripts/db-pg-migrate.sh
+++ b/docker/scripts/db-pg-migrate.sh
@@ -45,17 +45,21 @@ echo "Waiting for DB containers to come online..."
echo "Running initial checks..."
/usr/local/bin/python ./ietf/manage.py check --settings=settings_local
-# Migrate, adjusting to what the current state of the underlying database might be:
-/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
-
-# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines)
+# The mysql database is always freshly build container from the
+# image build of last-night's dump when this script is run
+# The first run of migrations will run anything merged from main that
+# that hasn't been released, and the few pre-engine-shift migrations
+# that the feat/postgres branch adds. It is guaranteed to fail at
+# utils.migrations.0004_pause_to_change_database_engines (where it
+# fails on purpose, hence the `|| true` so we may proceed
/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local || true
cat ./ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py ./ietf/settings_local.py
# Now transfer the migrated database from mysql to postgres unless that's already happened.
echo "Transferring migrated database from MySQL to PostgreSQL..."
-if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then
+EMPTY_CHECK=`psql -U django -h pgdb -d ietf -c "\dt" 2>&1`
+if echo ${EMPTY_CHECK} | grep -q "Did not find any relations."; then
cat << EOF > cast.load
LOAD DATABASE
FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8
@@ -65,6 +69,9 @@ EOF
time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load
rm cast.load
/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local
+else
+ echo "The postgres database is in an unexpected state"
+ echo ${EMPTY_CHECK}
fi
# Stop postgreSQL container
diff --git a/ietf/settings.py b/ietf/settings.py
index 144f321cc..7319501be 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -91,10 +91,6 @@ DATABASES = {
},
}
-DATABASE_TEST_OPTIONS = {
- # Comment this out if your database doesn't support InnoDB
- 'init_command': 'SET storage_engine=InnoDB',
-}
# Local time zone for this installation. Choices can be found here:
# http://www.postgresql.org/docs/8.1/static/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE
diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py
index 3a2251213..c053150d5 100644
--- a/ietf/utils/test_runner.py
+++ b/ietf/utils/test_runner.py
@@ -229,9 +229,6 @@ def safe_create_test_db(self, verbosity, *args, **kwargs):
keepdb = kwargs.get('keepdb', False)
if not keepdb:
print(" Creating test database...")
- if settings.DATABASES["default"]["ENGINE"] == 'django.db.backends.mysql':
- settings.DATABASES["default"]["OPTIONS"] = settings.DATABASE_TEST_OPTIONS
- print(" Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"])
test_database_name = old_create(self, 0, *args, **kwargs)
if settings.GLOBAL_TEST_FIXTURES: