chore: Prevent test suite artifact creation in work directory (#6438)

* chore: Prevent test suite artifact creation in work directory

Also remove a few other stale test assets while I'm here.

* Try and fix CI

* Change IDSUBMIT_REPOSITORY_PATH

* Make the dir

* test: clean up media/nomcom directories

* test: de-dup settings_temp_path_overrides

* chore: remove debug

* refactor: avoid premature import of test_utils

* refactor: drop useless lambda wrapper

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>

dev/tests/settings_local.py

@@ -17,8 +17,8 @@ DATABASES = {
 }
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_REPOSITORY_PATH = "test/id/"
+IDSUBMIT_REPOSITORY_PATH = "/assets/ietfdata/doc/draft/repository"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH

docker/configs/settings_local.py

@@ -8,7 +8,7 @@ ALLOWED_HOSTS = ['*']
 from ietf.settings_postgresqldb import DATABASES   # pyflakes:ignore
 
 IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
-IDSUBMIT_STAGING_PATH = "test/staging/"
+IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/"
 
 AGENDA_PATH = '/assets/www6s/proceedings/'
 MEETINGHOST_LOGO_PATH = AGENDA_PATH
@@ -53,7 +53,7 @@ IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH
 FTP_DIR = '/assets/ftp'
 
 NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
-SLIDE_STAGING_PATH = 'test/staging/'
+SLIDE_STAGING_PATH = '/assets/www6s/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 

docker/scripts/app-create-dirs.sh

@@ -1,13 +1,6 @@
 #!/bin/bash
 
 for sub in \
-    test/id \
-    test/staging \
-    test/archive \
-    test/rfc \
-    test/media \
-    test/wiki/ietf \
-    data/nomcom_keys/public_keys \
     /assets/archive/id \
     /assets/collection \
     /assets/collection/draft-archive \
@@ -27,6 +20,7 @@ for sub in \
     /assets/ietfdata/derived \
     /assets/ietfdata/derived/bibxml \
     /assets/ietfdata/derived/bibxml/bibxml-ids \
+    /assets/ietfdata/doc/draft/repository \
     /assets/www6s \
     /assets/www6s/staging \
     /assets/www6s/wg-descriptions \

ietf/meeting/tests_js.py

@@ -5,7 +5,7 @@
 import time
 import datetime
 import shutil
-import os
+import tempfile
 import re
 
 from django.utils import timezone
@@ -939,13 +939,8 @@ class InterimTests(IetfSeleniumTestCase):
     def tempdir(self, label):
         # Borrowed from  test_utils.TestCase
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
+        suffix = "-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
+        return tempfile.mkdtemp(suffix=suffix)
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
 
     def displayed_interims(self, groups=None):
         sessions = add_event_info_to_session_qs(

ietf/person/factories.py

@@ -8,7 +8,7 @@ import faker
 import faker.config
 import os
 import random
-import shutil
+from PIL import Image
 
 from unidecode import unidecode
 from unicodedata import normalize
@@ -103,10 +103,9 @@ class PersonFactory(factory.django.DjangoModelFactory):
             media_name = "%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name)
             obj.photo = media_name
             obj.photo_thumb = media_name
-            photosrc = os.path.join(settings.TEST_DATA_DIR, "profile-default.jpg")
             photodst = os.path.join(settings.PHOTOS_DIR,  photo_name + '.jpg')
-            if not os.path.exists(photodst):
+            img = Image.new('RGB', (200, 200))
-                shutil.copy(photosrc, photodst)
+            img.save(photodst)
             def delete_file(file):
                 os.unlink(file)
             atexit.register(delete_file, photodst)

ietf/settings_test.py

@@ -9,9 +9,12 @@
 #   ./manage.py test --settings=settings_test doc.ChangeStateTestCase
 #
 
-import os 
+import atexit
+import os
+import shutil
+import tempfile
 from ietf.settings import *                                          # pyflakes:ignore
-from ietf.settings import TEST_CODE_COVERAGE_CHECKER, BASE_DIR, PHOTOS_DIRNAME
+from ietf.settings import TEST_CODE_COVERAGE_CHECKER
 import debug                            # pyflakes:ignore
 debug.debug = True
 
@@ -48,11 +51,20 @@ DATABASES = {
 if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore
     TEST_CODE_COVERAGE_CHECKER.start()                          # pyflakes:ignore
 
-NOMCOM_PUBLIC_KEYS_DIR=os.path.abspath("tmp-nomcom-public-keys-dir")
 
-MEDIA_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'test/media/') # pyflakes:ignore
+def tempdir_with_cleanup(**kwargs):
-MEDIA_URL = '/test/media/'
+    """Utility to create a temporary dir and arrange cleanup"""
-PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME                            # pyflakes:ignore
+    _dir = tempfile.mkdtemp(**kwargs)
+    atexit.register(shutil.rmtree, _dir)
+    return _dir
+
+
+NOMCOM_PUBLIC_KEYS_DIR = tempdir_with_cleanup(suffix="-nomcom-public-keys-dir")
+
+MEDIA_ROOT = tempdir_with_cleanup(suffix="-media")
+PHOTOS_DIRNAME = "photo"
+PHOTOS_DIR = os.path.join(MEDIA_ROOT, PHOTOS_DIRNAME)
+os.mkdir(PHOTOS_DIR)
 
 # Undo any developer-dependent middleware when running the tests
 MIDDLEWARE = [ c for c in MIDDLEWARE if not c in DEV_MIDDLEWARE ] # pyflakes:ignore

ietf/utils/test_utils.py

@@ -34,7 +34,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
-import os
+import tempfile
 import re
 import email
 import html5lib
@@ -239,13 +239,8 @@ class TestCase(django.test.TestCase):
 
     def tempdir(self, label):
         slug = slugify(self.__class__.__name__.replace('.','-'))
-        dirname = "tmp-{label}-{slug}-dir".format(**locals())
+        suffix = "-{label}-{slug}-dir".format(**locals())
-        if 'VIRTUAL_ENV' in os.environ:
+        return tempfile.mkdtemp(suffix=suffix)
-            dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname)
-        path = os.path.abspath(dirname)
-        if not os.path.exists(path):
-            os.mkdir(path)
-        return path
 
     def assertNoFormPostErrors(self, response, error_css_selector=".is-invalid"):
         """Try to fish out form errors, if none found at least check the
@@ -306,7 +301,7 @@ class TestCase(django.test.TestCase):
 
         # Replace settings paths with temporary directories.
         self._ietf_temp_dirs = {}  # trashed during tearDown, DO NOT put paths you care about in this
-        for setting in self.settings_temp_path_overrides:
+        for setting in set(self.settings_temp_path_overrides):
             self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting))
         self._ietf_saved_context = django.test.utils.override_settings(**self._ietf_temp_dirs)
         self._ietf_saved_context.enable()

media/.gitignore

@@ -1 +0,0 @@
-/floor

test/data/youtube-playlistid.json

@@ -1 +0,0 @@
-{"regionCode": "US", "kind": "youtube#searchListResponse", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/eoV8llUEbIu5LXnwqBaLOkOK0Hg\"", "pageInfo": {"resultsPerPage": 1, "totalResults": 1}, "items": [{"snippet": {"thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/mqdefault.jpg", "width": 320, "height": 180}}, "title": "IETF98", "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-03-30T12:41:04.000Z", "liveBroadcastContent": "none", "channelTitle": "IETF - Internet Engineering Task Force", "description": "Videos from the IETF 98 Meeting held in Chicago, Illinois, United States 26-31 March 2017."}, "kind": "youtube#searchResult", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/X3dbZGRvgpvedtOP0KLGhZLg5UI\"", "id": {"kind": "youtube#playlist", "playlistId": "PLC86T-test"}}]}

test/data/youtube-playlistitems.json

@@ -1 +0,0 @@
-{"items": [{"snippet": {"playlistId": "PLC86T-6ZTP5jo6kIuqdyeYYhsKv9sUwG1", "thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/mqdefault.jpg", "width": 320, "height": 180}, "maxres": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/maxresdefault.jpg", "width": 1280, "height": 720}, "standard": {"url": "https://i.ytimg.com/vi/lhYWB5FFkg4/sddefault.jpg", "width": 640, "height": 480}}, "title": "IETF98 Wrap Up", "resourceId": {"kind": "youtube#video", "videoId": "lhYWB5FFkg4"}, "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-04-06T13:32:39.000Z", "channelTitle": "IETF - Internet Engineering Task Force", "position": 0, "description": "Jari Arkko and Alissa Cooper recap some highlights the IETF 98 meeting held 26-31 March 2017 in Chicago, Illinois, United States"}, "kind": "youtube#playlistItem", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/eW_De3gQF2fRzN_rPBbX-kY7oBI\"", "id": "UExDODZULTZaVFA1am82a0l1cWR5ZVlZaHNLdjlzVXdHMS40OTQ5QjlEMDgzN0FBNUIw"}, {"snippet": {"playlistId": "PLC86T-6ZTP5jo6kIuqdyeYYhsKv9sUwG1", "thumbnails": {"default": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/default.jpg", "width": 120, "height": 90}, "high": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/hqdefault.jpg", "width": 480, "height": 360}, "medium": {"url": "https://i.ytimg.com/vi/lPSTcBITbvs/mqdefault.jpg", "width": 320, "height": 180}}, "title": "IETF 98 - QUIC Tutorial", "resourceId": {"kind": "youtube#video", "videoId": "lPSTcBITbvs"}, "channelId": "UC8dtK9njBLdFnBahHFp0eZQ", "publishedAt": "2017-03-30T12:41:35.000Z", "channelTitle": "IETF - Internet Engineering Task Force", "position": 1, "description": "A tutorial about the new QUIC protocol"}, "kind": "youtube#playlistItem", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/GhKVt6zTuEpFavgtf9GWlWuzX9s\"", "id": "UExDODZULTZaVFA1am82a0l1cWR5ZVlZaHNLdjlzVXdHMS41NkI0NEY2RDEwNTU3Q0M2"}], "kind": "youtube#playlistItemListResponse", "etag": "\"m2yskBQFythfE4irbTIeOgYYfBU/jlFue-jZVpFMOuLUXQZH4Y0Lh3Y\"", "pageInfo": {"resultsPerPage": 2, "totalResults": 110}}

test/lib/.gitignore

@@ -1 +0,0 @@
-/django

test/lib/README

@@ -1,9 +0,0 @@
-This directory will be used to set up packages used for testing if they need any
-special handling which should not be applied to the system-wide setup.
-
-For instance, many of the tests to be run on the Django application should be run
-with a standard Django environment; but there are some Django test features which
-are broken in 0.9.6, and need patching in order to do the testing, like the ability
-to create a test database to run unit tests, according to this issue and patch:
-http://code.djangoproject.com/changeset/5106
-

test/media/floor/.gitignore

@@ -1 +0,0 @@
-/*

test/media/photo/.gitignore

@@ -1 +0,0 @@
-/*