feat: additional filesystem monitoring (#8405)

* feat: additional filesystem monitoring

* chore: rename setting for tmp directory

* fix: restructure path to new endpoint

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

dev/deploy-to-container/settings_local.py

@@ -64,6 +64,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/archive/id'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/test/staging/'

dev/diff/settings_local.py

@@ -60,6 +60,7 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/assets/collection/draft-archive'
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'

dev/tests/settings_local.py

@@ -59,6 +59,7 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/assets/collection/draft-archive'
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = 'test/staging/'

docker/configs/settings_local.py

@@ -50,6 +50,7 @@ INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/archive/id'
 BIBXML_BASE_PATH = '/assets/ietfdata/derived/bibxml'
 IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
+NFS_METRICS_TMP_DIR = '/assets/tmp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
 SLIDE_STAGING_PATH = '/assets/www6s/staging/'

docker/scripts/app-create-dirs.sh

@@ -29,6 +29,7 @@ for sub in \
     /assets/www6/iesg \
     /assets/www6/iesg/evaluation \
     /assets/media/photo \
+    /assets/tmp \
     /assets/ftp \
     /assets/ftp/charter \
     /assets/ftp/internet-drafts \

ietf/api/tests.py

@@ -970,6 +970,14 @@ class CustomApiTests(TestCase):
             self.assertEqual(jsondata['success'], True)
             self.client.logout()
 
+    @override_settings(APP_API_TOKENS={"ietf.api.views.nfs_metrics": ["valid-token"]})
+    def test_api_nfs_metrics(self):
+        url = urlreverse("ietf.api.views.nfs_metrics")
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 403)
+        r = self.client.get(url, headers={"X-Api-Key": "valid-token"})
+        self.assertContains(r, 'nfs_latency_seconds{operation="write"}')
+
     def test_api_get_session_matherials_no_agenda_meeting_url(self):
         meeting = MeetingFactory(type_id='ietf')
         session = SessionFactory(meeting=meeting)

ietf/api/urls.py

@@ -82,6 +82,8 @@ urlpatterns = [
     url(r'^version/?$', api_views.version),
     # Application authentication API key
     url(r'^appauth/(?P<app>authortools|bibxml)$', api_views.app_auth),
+    # NFS metrics endpoint
+    url(r'^metrics/nfs/?$', api_views.nfs_metrics),
     # latest versions
     url(r'^rfcdiff-latest-json/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, api_views.rfcdiff_latest_json),
     url(r'^rfcdiff-latest-json/(?P<name>[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json),

ietf/api/views.py

@@ -3,7 +3,10 @@
 
 import base64
 import binascii
+import datetime
 import json
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 import jsonschema
 import pytz
 import re
@@ -264,7 +267,22 @@ def app_auth(request, app: Literal["authortools", "bibxml"]):
             json.dumps({'success': True}),
             content_type='application/json')
 
-
+@requires_api_token
+@csrf_exempt
+def nfs_metrics(request):
+    with NamedTemporaryFile(dir=settings.NFS_METRICS_TMP_DIR,delete=False) as fp:
+        fp.close()
+        mark = datetime.datetime.now()
+        with open(fp.name, mode="w") as f:
+            f.write("whyioughta"*1024)
+        write_latency = (datetime.datetime.now() - mark).total_seconds()
+        mark = datetime.datetime.now()
+        with open(fp.name, "r") as f:
+            _=f.read()
+        read_latency = (datetime.datetime.now() - mark).total_seconds()
+        Path(f.name).unlink()
+    response=f'nfs_latency_seconds{{operation="write"}} {write_latency}\nnfs_latency_seconds{{operation="read"}} {read_latency}\n'
+    return HttpResponse(response)
 
 def find_doc_for_rfcdiff(name, rev):
     """rfcdiff lookup heuristics

ietf/settings.py

@@ -761,6 +761,7 @@ MEETING_RECORDINGS_DIR = '/a/www/audio'
 DERIVED_DIR = '/a/ietfdata/derived'
 FTP_DIR = '/a/ftp'
 ALL_ID_DOWNLOAD_DIR = '/a/www/www6s/download'
+NFS_METRICS_TMP_DIR = '/a/tmp'
 
 DOCUMENT_FORMAT_ALLOWLIST = ["txt", "ps", "pdf", "xml", "html", ]