997239a2ea
* feat: basic blobstore infrastructure for dev * refactor: (broken) attempt to put minio console behind nginx * feat: initialize blobstore with boto3 * fix: abandon attempt to proxy minio. Use docker compose instead. * feat: beginning of blob writes * feat: storage utilities * feat: test buckets * chore: black * chore: remove unused import * chore: avoid f string when not needed * fix: inform all settings files about blobstores * fix: declare types for some settings * ci: point to new target base * ci: adjust test workflow * fix: give the tests debug environment a blobstore * fix: "better" name declarations * ci: use devblobstore container * chore: identify places to write to blobstorage * chore: remove unreachable code * feat: store materials * feat: store statements * feat: store status changes * feat: store liaison attachments * feat: store agendas provided with Interim session requests * chore: capture TODOs * feat: store polls and chatlogs * chore: remove unneeded TODO * feat: store drafts on submit and post * fix: handle storage during doc expiration and resurrection * fix: mirror an unlink * chore: add/refine TODOs * feat: store slide submissions * fix: structure slide test correctly * fix: correct sense of existence check * feat: store some indexes * feat: BlobShadowFileSystemStorage * feat: shadow floorplans / host logos to the blob * chore: remove unused import * feat: strip path from blob shadow names * feat: shadow photos / thumbs * refactor: combine photo and photothumb blob kinds The photos / thumbs were already dropped in the same directory, so let's not add a distinction at this point. * style: whitespace * refactor: use kwargs consistently * chore: migrations * refactor: better deconstruct(); rebuild migrations * fix: use new class in mack patch * chore: add TODO * feat: store group index documents * chore: identify more TODO * feat: store reviews * fix: repair merge * chore: remove unnecessary TODO * feat: StoredObject metadata * fix: deburr some debugging code * fix: only set the deleted timestamp once * chore: correct typo * fix: get_or_create vs get and test * fix: avoid the questionable is_seekable helper * chore: capture future design consideration * chore: blob store cfg for k8s * chore: black * chore: copyright * ci: bucket name prefix option + run Black Adds/uses DATATRACKER_BLOB_STORE_BUCKET_PREFIX option. Other changes are just Black styling. * ci: fix typo in bucket name expression * chore: parameters in app-configure-blobstore Allows use with other blob stores. * ci: remove verify=False option * fix: don't return value from __init__ * feat: option to log timing of S3Storage calls * chore: units * fix: deleted->null when storing a file * style: Black * feat: log as JSON; refactor to share code; handle exceptions * ci: add ietf_log_blob_timing option for k8s * test: --no-manage-blobstore option for running tests * test: use blob store settings from env, if set * test: actually set a couple more storage opts * feat: offswitch (#8541) * feat: offswitch * fix: apply ENABLE_BLOBSTORAGE to BlobShadowFileSystemStorage behavior * chore: log timing of blob reads * chore: import Config from botocore.config * chore(deps): import boto3-stubs / botocore botocore is implicitly imported, but make it explicit since we refer to it directly * chore: drop type annotation that mypy loudly ignores * refactor: add storage methods via mixin Shares code between Document and DocHistory without putting it in the base DocumentInfo class, which lacks the name field. Also makes mypy happy. * feat: add timeout / retry limit to boto client * ci: let k8s config the timeouts via env * chore: repair merge resolution typo * chore: tweak settings imports * chore: simplify k8s/settings_local.py imports --------- Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
375 lines
14 KiB
Python
375 lines
14 KiB
Python
# Copyright The IETF Trust 2023, All Rights Reserved
|
|
|
|
import debug # pyflakes:ignore
|
|
|
|
from pyquery import PyQuery
|
|
|
|
from pathlib import Path
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from django.conf import settings
|
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
|
from django.template.loader import render_to_string
|
|
from django.urls import reverse as urlreverse
|
|
|
|
from ietf.doc.factories import StatementFactory, DocEventFactory
|
|
from ietf.doc.models import Document, State, NewRevisionDocEvent
|
|
from ietf.doc.storage_utils import retrieve_str
|
|
from ietf.group.models import Group
|
|
from ietf.person.factories import PersonFactory
|
|
from ietf.utils.mail import outbox, empty_outbox
|
|
from ietf.utils.test_utils import (
|
|
TestCase,
|
|
reload_db_objects,
|
|
login_testing_unauthorized,
|
|
)
|
|
|
|
|
|
class StatementsTestCase(TestCase):
|
|
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
|
|
"DOCUMENT_PATH_PATTERN"
|
|
]
|
|
|
|
def extract_content(self, response):
|
|
if not hasattr(response, "_cached_extraction"):
|
|
response._cached_extraction = list(response.streaming_content)[0].decode(
|
|
"utf-8"
|
|
)
|
|
return response._cached_extraction
|
|
|
|
def write_statement_markdown_file(self, statement):
|
|
(
|
|
Path(settings.DOCUMENT_PATH_PATTERN.format(doc=statement))
|
|
/ ("%s-%s.md" % (statement.name, statement.rev))
|
|
).write_text(
|
|
"""# This is a test statement.
|
|
Version: {statement.rev}
|
|
|
|
## A section
|
|
|
|
This test section has some text.
|
|
"""
|
|
)
|
|
|
|
def write_statement_pdf_file(self, statement):
|
|
(
|
|
Path(settings.DOCUMENT_PATH_PATTERN.format(doc=statement))
|
|
/ ("%s-%s.pdf" % (statement.name, statement.rev))
|
|
).write_text(
|
|
f"{statement.rev} This is not valid PDF, but the test does not need it to be"
|
|
)
|
|
|
|
def test_statement_doc_view(self):
|
|
doc = StatementFactory()
|
|
self.write_statement_markdown_file(doc)
|
|
url = urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))
|
|
response = self.client.get(url)
|
|
self.assertEqual(response.status_code, 200)
|
|
q = PyQuery(response.content)
|
|
self.assertEqual(q("#statement-state").text(), "Active")
|
|
self.assertEqual(q("#statement-type").text(), "IAB Statement")
|
|
self.assertIn("has some text", q(".card-body").text())
|
|
published = doc.docevent_set.filter(type="published_statement").last().time
|
|
self.assertIn(
|
|
published.astimezone(ZoneInfo(settings.TIME_ZONE)).date().isoformat(),
|
|
q("#published").text(),
|
|
)
|
|
|
|
doc.set_state(State.objects.get(type_id="statement", slug="replaced"))
|
|
doc2 = StatementFactory()
|
|
doc2.relateddocument_set.create(relationship_id="replaces", target=doc)
|
|
response = self.client.get(url)
|
|
self.assertEqual(response.status_code, 200)
|
|
q = PyQuery(response.content)
|
|
self.assertEqual(q("#statement-state").text(), "Replaced")
|
|
self.assertEqual(q("#statement-type").text(), "Replaced IAB Statement")
|
|
self.assertEqual(q("#statement-type").next().text(), f"Replaced by {doc2.name}")
|
|
|
|
url = urlreverse(
|
|
"ietf.doc.views_doc.document_main", kwargs=dict(name=doc2.name)
|
|
)
|
|
response = self.client.get(url)
|
|
self.assertEqual(response.status_code, 200)
|
|
q = PyQuery(response.content)
|
|
self.assertEqual(q("#statement-type").text(), "IAB Statement")
|
|
self.assertEqual(q("#statement-type").next().text(), f"Replaces {doc.name}")
|
|
|
|
def test_serve_pdf(self):
|
|
url = urlreverse(
|
|
"ietf.doc.views_statement.serve_pdf",
|
|
kwargs=dict(name="statement-does-not-exist"),
|
|
)
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 404)
|
|
|
|
doc = StatementFactory()
|
|
url = urlreverse(
|
|
"ietf.doc.views_statement.serve_pdf", kwargs=dict(name=doc.name)
|
|
)
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 404) # File not found
|
|
|
|
self.write_statement_pdf_file(doc)
|
|
doc.rev = "01"
|
|
e = DocEventFactory(type="published_statement", doc=doc, rev=doc.rev)
|
|
doc.save_with_history([e])
|
|
self.write_statement_pdf_file(doc)
|
|
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 200)
|
|
self.assertEqual(r.get("Content-Type"), "application/pdf")
|
|
self.assertTrue(
|
|
self.extract_content(r).startswith(doc.rev)
|
|
) # relies on test doc not actually being pdf
|
|
|
|
url = urlreverse(
|
|
"ietf.doc.views_statement.serve_pdf", kwargs=dict(name=doc.name, rev="00")
|
|
)
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 200)
|
|
self.assertTrue(self.extract_content(r).startswith("00 "))
|
|
url = urlreverse(
|
|
"ietf.doc.views_statement.serve_pdf", kwargs=dict(name=doc.name, rev="01")
|
|
)
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 200)
|
|
self.assertTrue(self.extract_content(r).startswith("01 "))
|
|
|
|
def test_submit(self):
|
|
doc = StatementFactory()
|
|
url = urlreverse("ietf.doc.views_statement.submit", kwargs=dict(name=doc.name))
|
|
|
|
rev = doc.rev
|
|
r = self.client.post(
|
|
url, {"statement_submission": "enter", "statement_content": "# oiwefrase"}
|
|
)
|
|
self.assertEqual(r.status_code, 302)
|
|
doc = reload_db_objects(doc)
|
|
self.assertEqual(rev, doc.rev)
|
|
|
|
nobody = PersonFactory()
|
|
self.client.login(
|
|
username=nobody.user.username, password=nobody.user.username + "+password"
|
|
)
|
|
r = self.client.post(
|
|
url, {"statement_submission": "enter", "statement_content": "# oiwefrase"}
|
|
)
|
|
self.assertEqual(r.status_code, 403)
|
|
doc = reload_db_objects(doc)
|
|
self.assertEqual(rev, doc.rev)
|
|
self.client.logout()
|
|
|
|
for username in ["secretary"]: # There is potential for expanding this list
|
|
self.client.login(username=username, password=username + "+password")
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 200)
|
|
file = SimpleUploadedFile(
|
|
"random.pdf",
|
|
b"not valid pdf",
|
|
content_type="application/pdf",
|
|
)
|
|
for postdict in [
|
|
{
|
|
"statement_submission": "enter",
|
|
"statement_content": f"# {username}",
|
|
},
|
|
{
|
|
"statement_submission": "upload",
|
|
"statement_file": file,
|
|
},
|
|
]:
|
|
docevent_count = doc.docevent_set.count()
|
|
empty_outbox()
|
|
r = self.client.post(url, postdict)
|
|
self.assertEqual(r.status_code, 302)
|
|
doc = reload_db_objects(doc)
|
|
self.assertEqual("%02d" % (int(rev) + 1), doc.rev)
|
|
if postdict["statement_submission"] == "enter":
|
|
self.assertEqual(f"# {username}", doc.text())
|
|
self.assertEqual(
|
|
retrieve_str("statement", f"{doc.name}-{doc.rev}.md"),
|
|
f"# {username}"
|
|
)
|
|
else:
|
|
self.assertEqual("not valid pdf", doc.text())
|
|
self.assertEqual(
|
|
retrieve_str("statement", f"{doc.name}-{doc.rev}.pdf"),
|
|
"not valid pdf"
|
|
)
|
|
self.assertEqual(docevent_count + 1, doc.docevent_set.count())
|
|
self.assertEqual(0, len(outbox))
|
|
rev = doc.rev
|
|
self.client.logout()
|
|
|
|
def test_start_new_statement(self):
|
|
url = urlreverse("ietf.doc.views_statement.new_statement")
|
|
login_testing_unauthorized(self, "secretary", url)
|
|
r = self.client.get(url)
|
|
self.assertContains(
|
|
r,
|
|
"Replace this with the content of the statement in markdown source",
|
|
status_code=200,
|
|
)
|
|
group = Group.objects.get(acronym="iab")
|
|
r = self.client.post(
|
|
url,
|
|
dict(
|
|
group=group.pk,
|
|
title="default",
|
|
statement_submission="enter",
|
|
statement_content=render_to_string(
|
|
"doc/statement/statement_template.md", {"settings": settings}
|
|
),
|
|
),
|
|
)
|
|
self.assertContains(r, "The example content may not be saved.", status_code=200)
|
|
|
|
file = SimpleUploadedFile(
|
|
"random.pdf",
|
|
b"not valid pdf",
|
|
content_type="application/pdf",
|
|
)
|
|
group = Group.objects.get(acronym="iab")
|
|
for postdict in [
|
|
dict(
|
|
group=group.pk,
|
|
title="title one",
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group=group.pk,
|
|
title="title two",
|
|
statement_submission="upload",
|
|
statement_file=file,
|
|
),
|
|
]:
|
|
empty_outbox()
|
|
r = self.client.post(url, postdict)
|
|
self.assertEqual(r.status_code, 302)
|
|
name = f"statement-{group.acronym}-{postdict['title']}".replace(
|
|
" ", "-"
|
|
) # cheap slugification
|
|
statement = Document.objects.filter(
|
|
name=name, type_id="statement"
|
|
).first()
|
|
self.assertIsNotNone(statement)
|
|
self.assertEqual(statement.title, postdict["title"])
|
|
self.assertEqual(statement.rev, "00")
|
|
self.assertEqual(statement.get_state_slug(), "active")
|
|
self.assertEqual(
|
|
statement.latest_event(NewRevisionDocEvent).rev, "00"
|
|
)
|
|
self.assertIsNotNone(statement.latest_event(type="published_statement"))
|
|
self.assertIsNotNone(statement.history_set.last().latest_event(type="published_statement"))
|
|
if postdict["statement_submission"] == "enter":
|
|
self.assertEqual(statement.text_or_error(), "some stuff")
|
|
self.assertEqual(
|
|
retrieve_str("statement", statement.uploaded_filename),
|
|
"some stuff"
|
|
)
|
|
else:
|
|
self.assertTrue(statement.uploaded_filename.endswith("pdf"))
|
|
self.assertEqual(
|
|
retrieve_str("statement", f"{statement.name}-{statement.rev}.pdf"),
|
|
"not valid pdf"
|
|
)
|
|
self.assertEqual(len(outbox), 0)
|
|
|
|
existing_statement = StatementFactory()
|
|
for postdict in [
|
|
dict(
|
|
group=group.pk,
|
|
title="",
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group=group.pk,
|
|
title="a title",
|
|
statement_submission="enter",
|
|
statement_content="",
|
|
),
|
|
dict(
|
|
group=group.pk,
|
|
title=existing_statement.title,
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group=group.pk,
|
|
title="森川",
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group=group.pk,
|
|
title="a title",
|
|
statement_submission="",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group="",
|
|
title="a title",
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
dict(
|
|
group=0,
|
|
title="a title",
|
|
statement_submission="enter",
|
|
statement_content="some stuff",
|
|
),
|
|
]:
|
|
r = self.client.post(url, postdict)
|
|
self.assertEqual(r.status_code, 200, f"Wrong status_code for {postdict}")
|
|
q = PyQuery(r.content)
|
|
self.assertTrue(
|
|
q("form div.is-invalid"), f"Expected an error for {postdict}"
|
|
)
|
|
|
|
def test_submit_non_markdown_formats(self):
|
|
doc = StatementFactory()
|
|
|
|
file = SimpleUploadedFile(
|
|
"random.pdf",
|
|
b"01 This is not valid PDF, but the test does not need it to be",
|
|
content_type="application/pdf",
|
|
)
|
|
|
|
url = urlreverse("ietf.doc.views_statement.submit", kwargs=dict(name=doc.name))
|
|
login_testing_unauthorized(self, "secretary", url)
|
|
|
|
r = self.client.post(
|
|
url,
|
|
{
|
|
"statement_submission": "upload",
|
|
"statement_file": file,
|
|
},
|
|
)
|
|
self.assertEqual(r.status_code, 302)
|
|
self.assertEqual(
|
|
r["Location"],
|
|
urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)),
|
|
)
|
|
|
|
doc = reload_db_objects(doc)
|
|
self.assertEqual(doc.rev, "01")
|
|
r = self.client.get(url)
|
|
self.assertEqual(r.status_code, 200)
|
|
q = PyQuery(r.content)
|
|
self.assertEqual(
|
|
q("#id_statement_content").text().strip(),
|
|
"The current revision of this statement is in pdf format",
|
|
)
|
|
|
|
file = SimpleUploadedFile(
|
|
"random.mp4", b"29ucdvn2o09hano5", content_type="video/mp4"
|
|
)
|
|
r = self.client.post(
|
|
url, {"statement_submission": "upload", "statement_file": file}
|
|
)
|
|
self.assertEqual(r.status_code, 200)
|
|
q = PyQuery(r.content)
|
|
self.assertTrue("Unexpected content" in q("#id_statement_file").next().text())
|