diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 93f94ff91..59d2344f5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -47,20 +47,9 @@ ], "sqltools.connections": [ // Default connection to dev DB container - { - "name": "Local MariaDB", - "server": "db", - "port": 3306, - "database": "ietf_utf8", - "username": "django", - "password": "RkTkDPFnKpko", - "driver": "MariaDB", - "askForPassword": false, - "connectionTimeout": 60 - }, { "name": "Local PostgreSQL", - "server": "pgdb", + "server": "db", "port": 5432, "database": "ietf", "username": "django", @@ -86,7 +75,6 @@ "ms-playwright.playwright", "ms-python.python", "ms-python.vscode-pylance", - "mtxr.sqltools-driver-mysql", "mtxr.sqltools-driver-pg", "mtxr.sqltools", "mutantdino.resourcemonitor", @@ -97,17 +85,13 @@ ], // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [8000, 3306], + "forwardPorts": [8000, 5432], "portsAttributes": { "3000": { "label": "Vite", "onAutoForward": "silent" }, - "3306": { - "label": "MariaDB", - "onAutoForward": "silent" - }, "5432": { "label": "PostgreSQL", "onAutoForward": "silent" diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 44649c117..ee226ede9 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -17,17 +17,10 @@ services: GITHUB_ACTIONS: 'true' HOME: /github/home db: - image: ghcr.io/ietf-tools/datatracker-db:latest + image: ghcr.io/ietf-tools/datatracker-db-pg:latest restart: unless-stopped volumes: - - mariadb-data:/var/lib/mysql - environment: - MYSQL_ROOT_PASSWORD: RkTkDPFnKpko - MYSQL_DATABASE: ietf_utf8 - MYSQL_USER: django - MYSQL_PASSWORD: RkTkDPFnKpko - CI: 'true' - GITHUB_ACTIONS: 'true' + - postgresdb-data:/var/lib/postgresql/data volumes: - mariadb-data: + postgresdb-data: diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index 16f36b6d8..73a6b8f94 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -8,19 +8,14 @@ ALLOWED_HOSTS = ['*'] DATABASES = { 'default': { 'HOST': 'db', - 'PORT': 3306, - 'NAME': 'ietf_utf8', - 'ENGINE': 'django.db.backends.mysql', + 'PORT': 5432, + 'NAME': 'ietf', + 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', - 'OPTIONS': { - 'sql_mode': 'STRICT_TRANS_TABLES', - 'init_command': 'SET storage_engine=InnoDB; SET names "utf8"', - }, }, } - IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_REPOSITORY_PATH = "test/id/" IDSUBMIT_STAGING_PATH = "test/staging/" diff --git a/docker-compose.yml b/docker-compose.yml index b25a64ef9..cb2a58986 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,35 +35,11 @@ services: # (Adding the "ports" property to this file will not forward from a Codespace.) db: - image: ghcr.io/ietf-tools/datatracker-db:latest + image: ghcr.io/ietf-tools/datatracker-db-pg:latest # build: # context: .. # dockerfile: docker/db.Dockerfile restart: unless-stopped - volumes: - - mariadb-data:/var/lib/mysql - environment: - MYSQL_ROOT_PASSWORD: RkTkDPFnKpko - MYSQL_DATABASE: ietf_utf8 - MYSQL_USER: django - MYSQL_PASSWORD: RkTkDPFnKpko - command: - - '--character-set-server=utf8' - - '--collation-server=utf8_unicode_ci' - - '--innodb-buffer-pool-size=1G' - - '--innodb-log-buffer-size=128M' - - '--innodb-log-file-size=256M' - - '--innodb-write-io-threads=8' - - '--innodb-flush-log-at-trx-commit=0' - - '--performance-schema=1' - - pgdb: - build: - context: . - dockerfile: docker/pgdb.Dockerfile - restart: unless-stopped - environment: - POSTGRES_PASSWORD: hk2j22sfiv volumes: - postgresdb-data:/var/lib/postgresql/data @@ -112,5 +88,4 @@ services: volumes: postgresdb-data: - mariadb-data: app-assets: diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index cee35097e..759bf6b7f 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -5,8 +5,7 @@ from ietf.settings import * # pyflakes: ALLOWED_HOSTS = ['*'] -from ietf.settings_mysqldb import DATABASES - +from ietf.settings_postgresqldb import DATABASES IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_REPOSITORY_PATH = "test/id/" diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py index e400420b5..5f62a5a2c 100644 --- a/docker/configs/settings_postgresqldb.py +++ b/docker/configs/settings_postgresqldb.py @@ -1,6 +1,6 @@ DATABASES = { 'default': { - 'HOST': 'pgdb', + 'HOST': 'db', 'PORT': 5432, 'NAME': 'ietf', 'ENGINE': 'django.db.backends.postgresql_psycopg2', diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index 06e47bbb0..12df388c4 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -14,7 +14,7 @@ services: - app-assets:/assets db: ports: - - '3306' + - '5432' celery: volumes: - .:/workspace diff --git a/docker/pgdb.Dockerfile b/docker/pgdb.Dockerfile deleted file mode 100644 index e566ab645..000000000 --- a/docker/pgdb.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM postgres:14.5 -LABEL maintainer="IETF Tools Team " - -#RUN apt-get update \ -# && apt-get install -y --no-install-recommends \ -# postgresql-14-pg-catcheck \ -# postgresql-14-powa \ -# postgresql-14-pg-qualstats \ -# postgresql-14-pg-stat-kcache \ -# postgresql-14-pg-stat-monitor \ -# postgresql-14-pg-top \ -# postgresql-14-pg-track_settings \ -# postgresql-14-pg-wait_sampling \ -# pgsql_tweaks - -ENV POSTGRES_PASSWORD=hk2j22sfiv -ENV POSTGRES_HOST_AUTH_METHOD=trust - -COPY docker/scripts/pgdb-ietf-init.sh /docker-entrypoint-initdb.d/ diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index f4fe42948..83f8dccdc 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -28,12 +28,8 @@ yarn build yarn legacy:build # Copy config files if needed -if [ ! -f "$WORKSPACEDIR/ietf/settings_mysqldb.py" ]; then - cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py -fi -if [ ! -f "$WORKSPACEDIR/ietf/settings_postgresqldb.py" ]; then - cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py -fi +cp $WORKSPACEDIR/docker/configs/settings_mysqldb.py $WORKSPACEDIR/ietf/settings_mysqldb.py +cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/settings_postgresqldb.py if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then echo "Setting up a default settings_local.py ..." @@ -81,8 +77,8 @@ else fi # Recondition settings to make changing databases easier. (Remember that we may have developers starting with settings_local from earlier work) - python docker/scripts/db-include-fix.py - cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_postgresqldb import DATABASES/from ietf.settings_mysqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py +python docker/scripts/db-include-fix.py +cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py # Create data directories @@ -99,10 +95,12 @@ curl -fsSL https://github.com/ietf-tools/datatracker/releases/download/baseline/ if [ -n "$EDITOR_VSCODE" ]; then echo "Waiting for DB container to come online ..." - /usr/local/bin/wait-for db:3306 -- echo "MariaDB ready" - /usr/local/bin/wait-for pgdb:5432 -- echo "Postgresql ready" + /usr/local/bin/wait-for db:5432 -- echo "PostgreSQL ready" fi +# Make sure PG search path is set +psql -U django -h db -d ietf -v ON_ERROR_STOP=1 -c '\x' -c 'ALTER USER django set search_path=ietf_utf8,django,public;' + # Run memcached echo "Starting memcached..." @@ -117,29 +115,6 @@ echo "Running initial checks..." /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local - -# We may be starting with a post 9.0.0 deploy dump, so run the migrations again before switching engines to catch any pre-postgres migrations that may be merged in from main post 9.0.0 (and any that are specific to feat/postgres that need to run before we switch engines) -/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local || true - -echo "DEBUGGING pt 2 - this should say mysqldb" -grep "DATA" $WORKSPACEDIR/ietf/settings_local.py -cat $WORKSPACEDIR/ietf/settings_local.py | sed 's/from ietf.settings_mysqldb import DATABASES/from ietf.settings_postgresqldb import DATABASES/' > /tmp/settings_local.py && mv /tmp/settings_local.py $WORKSPACEDIR/ietf/settings_local.py -echo "DEBUGGING pt 3 - this should say postgresdb" -grep "DATA" $WORKSPACEDIR/ietf/settings_local.py - -# Now transfer the migrated database from mysql to postgres unless that's already happened. -if psql -U django -h pgdb -d ietf -c "\dt" 2>&1 | grep -q "Did not find any relations."; then - cat << EOF > cast.load -LOAD DATABASE -FROM mysql://django:RkTkDPFnKpko@db/ietf_utf8 -INTO postgresql://django:RkTkDPFnKpko@pgdb/ietf -CAST type varchar to text drop typemod; -EOF - time pgloader --verbose --logfile=ietf_pgloader.run --summary=ietf_pgloader.summary cast.load - rm cast.load - /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local -fi - echo "-----------------------------------------------------------------" echo "Done!" echo "-----------------------------------------------------------------" diff --git a/docker/scripts/db-include-fix.py b/docker/scripts/db-include-fix.py index 369867061..05336641b 100644 --- a/docker/scripts/db-include-fix.py +++ b/docker/scripts/db-include-fix.py @@ -15,7 +15,7 @@ newcontent = content.replace( }, }, }""", - "from ietf.settings_mysqldb import DATABASES", + "from ietf.settings_postgresqldb import DATABASES", ).replace( """DATABASES = { 'default': { diff --git a/docker/scripts/pgdb-ietf-init.sh b/docker/scripts/pgdb-ietf-init.sh deleted file mode 100755 index 533084056..000000000 --- a/docker/scripts/pgdb-ietf-init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -psql -v ON_ERROR_STOP=1 --username postgres --dbname postgres <<-EOSQL - CREATE USER django PASSWORD 'RkTkDPFnKpko'; - CREATE DATABASE ietf; - GRANT ALL PRIVILEGES ON DATABASE ietf TO django; - ALTER USER django set search_path=ietf_utf8,django,public; -EOSQL