chore: adjustments for k8s deployment (#7531)

* chore: handle TERM in datatracker-start.sh

* chore: delay celery start if migration needed

* chore: skip-checks when migrating

* chore: label beat/celery as deleteBeforeUpgrade

Used by the infra-k8s deployment process to flag
these as needing to be shut down before a new
release rolls out.

* chore: increase termination grace periods

dev/build/celery-start.sh

@@ -5,6 +5,14 @@
 echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
+if ! ietf/manage.py migrate --skip-checks --check ; then
+    echo "Unapplied migrations found, waiting to start..."
+    sleep 5
+    while ! ietf/manage.py migrate --skip-checks --check ; do 
+        sleep 5
+    done
+fi
+
 cleanup () {
   # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
   if [[ -n "${celery_pid}" ]]; then

dev/build/datatracker-start.sh

@@ -4,14 +4,29 @@ echo "Running Datatracker checks..."
 ./ietf/manage.py check
 
 echo "Running Datatracker migrations..."
-./ietf/manage.py migrate --settings=settings_local
+./ietf/manage.py migrate --skip-checks --settings=settings_local
 
 echo "Starting Datatracker..."
 
+# trap TERM and shut down gunicorn
+cleanup () {
+    if [[ -n "${gunicorn_pid}" ]]; then
+        echo "Terminating gunicorn..."
+        kill -TERM "${gunicorn_pid}"
+        wait "${gunicorn_pid}"
+    fi
+}
+
+trap 'trap "" TERM; cleanup' TERM
+
+# start gunicorn in the background so we can trap the TERM signal
 gunicorn \
           --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
           --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
           --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
           --bind :8000 \
           --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
-          ietf.wsgi:application
+          ${DATATRACKER_GUNICORN_EXTRA_ARGS} \
+          ietf.wsgi:application &
+gunicorn_pid=$!
+wait "${gunicorn_pid}"

k8s/beat.yaml

@@ -2,6 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: beat
+  labels:
+    deleteBeforeUpgrade: yes
 spec:
   replicas: 1
   revisionHistoryLimit: 2
@@ -58,4 +60,4 @@ spec:
             name: files-cfgmap
       dnsPolicy: ClusterFirst
       restartPolicy: Always
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 600

k8s/celery.yaml

@@ -2,6 +2,8 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: celery
+  labels:
+    deleteBeforeUpgrade: yes
 spec:
   replicas: 1
   revisionHistoryLimit: 2
@@ -77,4 +79,4 @@ spec:
             name: files-cfgmap
       dnsPolicy: ClusterFirst
       restartPolicy: Always
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 600

k8s/datatracker.yaml

@@ -77,7 +77,7 @@ spec:
             name: files-cfgmap
       dnsPolicy: ClusterFirst
       restartPolicy: Always
-      terminationGracePeriodSeconds: 30
+      terminationGracePeriodSeconds: 60
 ---
 apiVersion: v1
 kind: Service