feat: add nginx, robots.txt, HTTP headers (#7683)

* feat: nginx + robots.txt

* feat: minimal /health/ endpoint

* ci: startupProbe for datatracker pod

* ci: probe auth pod; set timeoutSeconds

* feat: add CSP and other headers to nginx

* fix: typo in nginx.conf

* feat: split auth/dt nginx confs

* test: test health endpoint

* ci: auth service on port 80

We'll remove http-old (8080) in the future.

* ci: rename auth container/nginx cfg

ietf/urls.py

@@ -5,6 +5,7 @@ from django.conf.urls.static import static as static_url
 from django.contrib import admin
 from django.contrib.sitemaps import views as sitemap_views
 from django.contrib.staticfiles.urls import staticfiles_urlpatterns
+from django.http import HttpResponse
 from django.urls import include, path
 from django.views import static as static_view
 from django.views.generic import TemplateView
@@ -35,6 +36,7 @@ sitemaps = {
 
 urlpatterns = [
     url(r'^$', views_search.frontpage),
+    url(r'^health/', lambda _: HttpResponse()),
     url(r'^accounts/', include('ietf.ietfauth.urls')),
     url(r'^admin/', admin.site.urls),
     url(r'^admin/docs/', include('django.contrib.admindocs.urls')),

ietf/utils/tests.py

@@ -679,3 +679,12 @@ class SearchableFieldTests(TestCase):
         self.assertTrue(changed_form.has_changed())
         unchanged_form = TestForm(initial={'test_field': [1]}, data={'test_field': [1]})
         self.assertFalse(unchanged_form.has_changed())
+
+
+class HealthTests(TestCase):
+    def test_health(self):
+        self.assertEqual(
+            self.client.get("/health/").status_code,
+            200,
+        )
+            

k8s/auth.yaml

@@ -24,10 +24,6 @@ spec:
         - name: auth
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -49,6 +45,14 @@ spec:
           envFrom:
             - secretRef:
                 name: dt-secrets-env
+          startupProbe:
+            httpGet:
+              port: 8000
+              path: /health/
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+            timeoutSeconds: 3
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -58,6 +62,28 @@ spec:
             runAsUser: 1000
             runAsGroup: 1000
         # -----------------------------------------------------
+        # Nginx Container
+        # -----------------------------------------------------
+        - name: nginx
+          image: "ghcr.io/nginxinc/nginx-unprivileged:1.27"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              port: 8080
+              path: /health/nginx
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-tmp
+              mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/auth.conf
+              subPath: nginx-auth.conf
+        # -----------------------------------------------------
         # ScoutAPM Container
         # -----------------------------------------------------
         - name: scoutapm
@@ -97,6 +123,9 @@ spec:
         - name: dt-cfg
           configMap:
             name: files-cfgmap
+        - name: nginx-tmp
+          emptyDir:
+            sizeLimit: "500Mi"
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 60
@@ -108,9 +137,13 @@ metadata:
 spec:
   type: ClusterIP
   ports:
-    - port: 8080
+    - port: 80
       targetPort: http
       protocol: TCP
       name: http
+    - port: 8080
+      targetPort: http
+      protocol: TCP
+      name: http-old
   selector:
     app: auth

k8s/datatracker.yaml

@@ -24,10 +24,6 @@ spec:
         - name: datatracker
           image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
           imagePullPolicy: Always
-          ports:
-            - containerPort: 8000
-              name: http
-              protocol: TCP
           volumeMounts:
             - name: dt-vol
               mountPath: /a
@@ -49,6 +45,14 @@ spec:
           envFrom:
             - secretRef:
                 name: dt-secrets-env
+          startupProbe:
+            httpGet:
+              port: 8000
+              path: /health/
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+            timeoutSeconds: 3
           securityContext:
             allowPrivilegeEscalation: false
             capabilities:
@@ -58,6 +62,28 @@ spec:
             runAsUser: 1000
             runAsGroup: 1000
         # -----------------------------------------------------
+        # Nginx Container
+        # -----------------------------------------------------
+        - name: nginx
+          image: "ghcr.io/nginxinc/nginx-unprivileged:1.27"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+              name: http
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              port: 8080
+              path: /health/nginx
+          securityContext:
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: nginx-tmp
+              mountPath: /tmp
+            - name: dt-cfg
+              mountPath: /etc/nginx/conf.d/datatracker.conf
+              subPath: nginx-datatracker.conf
+        # -----------------------------------------------------
         # ScoutAPM Container
         # -----------------------------------------------------
         - name: scoutapm
@@ -126,6 +152,9 @@ spec:
         - name: dt-cfg
           configMap:
             name: files-cfgmap
+        - name: nginx-tmp
+          emptyDir:
+            sizeLimit: "500Mi"
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       terminationGracePeriodSeconds: 60

k8s/kustomization.yaml

@@ -3,6 +3,8 @@ namePrefix: dt-
 configMapGenerator:
   - name: files-cfgmap
     files:
+      - nginx-auth.conf
+      - nginx-datatracker.conf
       - settings_local.py
 resources:
   - auth.yaml

k8s/nginx-auth.conf

@@ -0,0 +1,34 @@
+server {
+    listen 8080 default_server;
+    server_name _;
+
+    # Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that
+    # our deny all rule does not squelch the other locations. 
+    location ~ ^/health/nginx$ {
+        return 200;
+    }
+
+    location ~ ^/robots.txt$ {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: *\nDisallow: /\n";
+    }
+
+    location ~ ^/accounts/create.* {
+        return 302 https://datatracker.ietf.org/accounts/create;
+    }
+
+    # n.b. (?!...) is a negative lookahead group
+    location ~ ^(/(?!(api/openid/|accounts/login/|accounts/logout/|accounts/reset/|person/.*/photo|group/groupmenu.json)).*) {
+        deny all;
+    }
+
+    location / {
+        add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com gather.town *.gather.town";
+        proxy_set_header Host $${keepempty}host;
+        proxy_set_header Connection close;
+        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_pass http://localhost:8000;
+    }
+}

k8s/nginx-datatracker.conf

@@ -0,0 +1,23 @@
+server {
+    listen 8080 default_server;
+    server_name _;
+
+    location /health/nginx {
+        return 200;
+    }
+
+    location /robots.txt {
+        add_header Content-Type text/plain;
+        return 200 "User-agent: *\nDisallow: /doc/pdf/\n";
+    }
+
+    location / {
+        add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com";
+        proxy_set_header Host $${keepempty}host;
+        proxy_set_header Connection close;
+        proxy_set_header X-Request-Start "t=${msec}";
+        proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
+        proxy_set_header X-Real-IP $${keepempty}remote_addr;
+        proxy_pass http://localhost:8000;
+    }
+}