diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 28b549537..4a0849f5a 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -3,7 +3,6 @@ import debug # pyflakes:ignore import io -import markdown from django import forms from django.contrib.auth.decorators import login_required @@ -20,6 +19,7 @@ from ietf.doc.utils import add_state_change_event from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.ietfauth.utils import has_role, role_required from ietf.person.fields import SearchablePersonsField +from ietf.utils import markdown from ietf.utils.response import permission_denied from ietf.utils.text import xslugify from ietf.utils.textupload import get_cleaned_text_file_content @@ -64,7 +64,7 @@ class BofreqUploadForm(forms.Form): if require_field("bofreq_file"): content = get_cleaned_text_file_content(self.cleaned_data["bofreq_file"]) try: - _ = markdown.markdown(content, extensions=['extra']) + _ = markdown.markdown(content) except Exception as e: raise forms.ValidationError(f'Markdown processing failed: {e}') diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index f2ccb3ec9..64c63e5e9 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -40,7 +40,6 @@ import io import json import os import re -import markdown from urllib.parse import quote @@ -80,7 +79,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc -from ietf.utils import markup_txt, log +from ietf.utils import markup_txt, log, markdown from ietf.utils.draft import Draft from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split @@ -550,7 +549,7 @@ def document_main(request, name, rev=None): )) if doc.type_id == "bofreq": - content = markdown.markdown(doc.text_or_error(),extensions=['extra']) + content = markdown.markdown(doc.text_or_error()) editors = bofreq_editors(doc) responsible = bofreq_responsible(doc) can_manage = has_role(request.user,['Secretariat', 'Area Director', 'IAB']) @@ -661,7 +660,7 @@ def document_main(request, name, rev=None): content = doc.text_or_error() t = "plain text" elif extension == ".md": - content = markdown.markdown(doc.text_or_error(), extensions=['extra']) + content = markdown.markdown(doc.text_or_error()) content_is_html = True t = "markdown" other_types.append((t, url)) diff --git a/ietf/group/views.py b/ietf/group/views.py index 0cfd38259..3ff48f5ba 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -38,7 +38,6 @@ import copy import datetime import itertools import io -import markdown import math import os import re @@ -121,7 +120,7 @@ from ietf.settings import MAILING_LIST_INFO_URL from ietf.utils.pipe import pipe from ietf.utils.response import permission_denied from ietf.utils.text import strip_suffix - +from ietf.utils import markdown # --- Helpers ---------------------------------------------------------- @@ -581,7 +580,7 @@ def group_about_rendertest(request, acronym, group_type=None): if group.charter: charter = get_charter_text(group) try: - rendered = markdown.markdown(charter, extensions=['extra']) + rendered = markdown.markdown(charter) except Exception as e: rendered = f'Markdown rendering failed: {e}' return render(request, 'group/group_about_rendertest.html', {'group':group, 'charter':charter, 'rendered':rendered}) diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 156b11d22..2158a5e94 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -436,6 +436,10 @@ class UploadSlidesForm(ApplyToAllFileUploadForm): return title +class ImportMinutesForm(forms.Form): + markdown_text = forms.CharField(strip=False, widget=forms.HiddenInput) + + class RequestMinutesForm(forms.Form): to = MultiEmailField() cc = MultiEmailField(required=False) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 7abe1b1c6..021457cf2 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -14,6 +14,7 @@ import string from collections import namedtuple from pathlib import Path +from urllib.parse import urljoin import debug # pyflakes:ignore @@ -1260,6 +1261,13 @@ class Session(models.Model): else: return self.group.acronym + def notes_id(self): + note_id_fragment = 'plenary' if self.type.slug == 'plenary' else self.group.acronym + return f'notes-ietf-{self.meeting.number}-{note_id_fragment}' + + def notes_url(self): + return urljoin(settings.IETF_NOTES_URL, self.notes_id()) + class SchedulingEvent(models.Model): session = ForeignKey(Session) time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened") diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 450c7e1e4..a466955fc 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -8,6 +8,8 @@ import random import re import shutil import pytz +import requests.exceptions +import requests_mock from unittest import skipIf from mock import patch, PropertyMock @@ -19,7 +21,6 @@ from urllib.parse import urlparse, urlsplit from PIL import Image from pathlib import Path - from django.urls import reverse as urlreverse from django.conf import settings from django.contrib.auth.models import User @@ -5405,12 +5406,16 @@ class IphoneAppJsonTests(TestCase): self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists()) class FinalizeProceedingsTests(TestCase): - @patch('urllib.request.urlopen') - def test_finalize_proceedings(self, mock_urlopen): - mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}') + @requests_mock.Mocker() + def test_finalize_proceedings(self, mock): make_meeting_test_data() meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last() meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None) + mock.get( + settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number), + text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]), + ) url = urlreverse('ietf.meeting.views.finalize_proceedings',kwargs={'num':meeting.number}) login_testing_unauthorized(self,"secretary",url) @@ -5605,8 +5610,10 @@ class MaterialsTests(TestCase): self.assertEqual(doc.rev,'02') # Verify that we don't have dead links - url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) + url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) top = '/meeting/%s/' % session.meeting.number + self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes') + self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'})) self.crawl_materials(url=url, top=top) def test_upload_minutes_agenda_unscheduled(self): @@ -5653,8 +5660,10 @@ class MaterialsTests(TestCase): self.assertEqual(doc.rev,'00') # Verify that we don't have dead links - url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) + url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym}) top = '/meeting/%s/' % session.meeting.number + self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes') + self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'})) self.crawl_materials(url=url, top=top) def test_upload_slides(self): @@ -5928,6 +5937,151 @@ class MaterialsTests(TestCase): self.assertIn('third version', contents) +@override_settings(IETF_NOTES_URL='https://notes.ietf.org/') +class ImportNotesTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH'] + + def setUp(self): + super().setUp() + self.session = SessionFactory(meeting__type_id='ietf') + self.meeting = self.session.meeting + + def test_retrieves_note(self): + """Can import and preview a note from notes.ietf.org""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + with requests_mock.Mocker() as mock: + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text') + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', + text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"})) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + iframe = q('iframe#preview') + self.assertEqual('

markdown text

', iframe.attr('srcdoc')) + markdown_text_input = q('form #id_markdown_text') + self.assertEqual(markdown_text_input.val(), 'markdown text') + + def test_retrieves_with_broken_metadata(self): + """Can import and preview a note even if it has a metadata problem""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + with requests_mock.Mocker() as mock: + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text') + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', text='this is not valid json {]') + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + iframe = q('iframe#preview') + self.assertEqual('

markdown text

', iframe.attr('srcdoc')) + markdown_text_input = q('form #id_markdown_text') + self.assertEqual(markdown_text_input.val(), 'markdown text') + + def test_redirects_on_success(self): + """Redirects to session details page after import""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + r = self.client.post(url, {'markdown_text': 'markdown text'}) + self.assertRedirects( + r, + urlreverse( + 'ietf.meeting.views.session_details', + kwargs={ + 'num': self.meeting.number, + 'acronym': self.session.group.acronym, + }, + ), + ) + + def test_imports_previewed_text(self): + """Import text that was shown as preview even if notes site is updated""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + with requests_mock.Mocker() as mock: + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='updated markdown text') + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', + text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"})) + r = self.client.post(url, {'markdown_text': 'original markdown text'}) + self.assertEqual(r.status_code, 302) + minutes_path = Path(self.meeting.get_materials_path()) / 'minutes' + with (minutes_path / self.session.minutes().uploaded_filename).open() as f: + self.assertEqual(f.read(), 'original markdown text') + + def test_refuses_identical_import(self): + """Should not be able to import text identical to the current revision""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + r = self.client.post(url, {'markdown_text': 'original markdown text'}) # create a rev + self.assertEqual(r.status_code, 302) + with requests_mock.Mocker() as mock: + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text') + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', + text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"})) + r = self.client.get(url) # try to import the same text + self.assertContains(r, "This document is identical", status_code=200) + q = PyQuery(r.content) + self.assertEqual(len(q('button:disabled[type="submit"]')), 1) + self.assertEqual(len(q('button:not(:disabled)[type="submit"]')), 0) + + def test_handles_missing_previous_revision_file(self): + """Should still allow import if the file for the previous revision is missing""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + r = self.client.post(url, {'markdown_text': 'original markdown text'}) # create a rev + # remove the file uploaded for the first rev + minutes_docs = self.session.sessionpresentation_set.filter(document__type='minutes') + self.assertEqual(minutes_docs.count(), 1) + Path(minutes_docs.first().document.get_file_name()).unlink() + + self.assertEqual(r.status_code, 302) + with requests_mock.Mocker() as mock: + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text') + mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', + text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"})) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + iframe = q('iframe#preview') + self.assertEqual('

original markdown text

', iframe.attr('srcdoc')) + markdown_text_input = q('form #id_markdown_text') + self.assertEqual(markdown_text_input.val(), 'original markdown text') + + def test_handles_note_does_not_exist(self): + """Should not try to import a note that does not exist""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + + self.client.login(username='secretary', password='secretary+password') + with requests_mock.Mocker() as mock: + mock.get(requests_mock.ANY, status_code=404) + r = self.client.get(url, follow=True) + self.assertContains(r, 'Could not import', status_code=200) + + def test_handles_notes_server_failure(self): + """Problems communicating with the notes server should be handled gracefully""" + url = urlreverse('ietf.meeting.views.import_session_minutes', + kwargs={'num': self.meeting.number, 'session_id': self.session.pk}) + self.client.login(username='secretary', password='secretary+password') + + with requests_mock.Mocker() as mock: + mock.get(re.compile(r'.+/download'), exc=requests.exceptions.ConnectTimeout) + mock.get(re.compile(r'.+//info'), text='{}') + r = self.client.get(url, follow=True) + self.assertContains(r, 'Could not reach the notes server', status_code=200) + + class SessionTests(TestCase): def test_meeting_requests(self): @@ -6911,12 +7065,15 @@ class ProceedingsTests(BaseMeetingTestCase): 0, ) - @patch('ietf.meeting.utils.requests.get') - def test_proceedings_attendees(self, mockobj): - mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]' - mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}') + @requests_mock.Mocker() + def test_proceedings_attendees(self, mock): make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97") + mock.get( + settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number), + text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]), + ) finalize(meeting) url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':97}) response = self.client.get(url) @@ -6924,14 +7081,18 @@ class ProceedingsTests(BaseMeetingTestCase): q = PyQuery(response.content) self.assertEqual(1,len(q("#id_attendees tbody tr"))) - @patch('urllib.request.urlopen') - def test_proceedings_overview(self, mock_urlopen): + @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}') + @requests_mock.Mocker() + def test_proceedings_overview(self, mock): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. ''' - mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97") + mock.get( + settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number), + text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]), + ) finalize(meeting) url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':97}) response = self.client.get(url) diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index f4bf41ec5..60f8ba8d8 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -13,6 +13,7 @@ safe_for_all_meeting_types = [ url(r'^session/(?P\d+)/bluesheets$', views.upload_session_bluesheets), url(r'^session/(?P\d+)/minutes$', views.upload_session_minutes), url(r'^session/(?P\d+)/agenda$', views.upload_session_agenda), + url(r'^session/(?P\d+)/import/minutes$', views.import_session_minutes), url(r'^session/(?P\d+)/propose_slides$', views.propose_session_slides), url(r'^session/(?P\d+)/slides(?:/%(name)s)?$' % settings.URL_REGEXPS, views.upload_session_slides), url(r'^session/(?P\d+)/add_to_session$', views.ajax_add_slides_to_session), diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 6ac8e4c10..6b6174191 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1,17 +1,19 @@ # Copyright The IETF Trust 2016-2020, All Rights Reserved # -*- coding: utf-8 -*- - - import datetime import itertools import re import requests +import subprocess from collections import defaultdict +from pathlib import Path from urllib.error import HTTPError from django.conf import settings +from django.contrib import messages from django.template.loader import render_to_string +from django.utils.encoding import smart_text from django.utils.html import format_html from django.utils.safestring import mark_safe @@ -19,11 +21,14 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate from ietf.meeting.models import Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment +from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName from ietf.person.models import Person from ietf.secr.proceedings.proc_utils import import_audio_files +from ietf.utils.html import sanitize_document + def session_time_for_sorting(session, use_meeting_date): official_timeslot = TimeSlot.objects.filter(sessionassignments__session=session, sessionassignments__schedule__in=[session.meeting.schedule, session.meeting.schedule.base if session.meeting.schedule else None]).first() @@ -544,3 +549,179 @@ def preprocess_meeting_important_dates(meetings): for d in m.important_dates: d.midnight_cutoff = "UTC 23:59" in d.name.name + +def get_meeting_sessions(num, acronym): + types = ['regular','plenary','other'] + sessions = Session.objects.filter( + meeting__number=num, + group__acronym=acronym, + type__in=types, + ) + if not sessions: + sessions = Session.objects.filter( + meeting__number=num, + short=acronym, + type__in=types, + ) + return sessions + + +class SessionNotScheduledError(Exception): + """Indicates failure because operation requires a scheduled session""" + pass + + +class SaveMaterialsError(Exception): + """Indicates failure saving session materials""" + pass + + +def save_session_minutes_revision(session, file, ext, request, encoding=None, apply_to_all=False): + """Creates or updates session minutes records + + This updates the database models to reflect a new version. It does not handle + storing the new file contents, that should be handled via handle_upload_file() + or similar. + + If the session does not already have minutes, it must be a scheduled + session. If not, SessionNotScheduledError will be raised. + + Returns (Document, [DocEvents]), which should be passed to doc.save_with_history() + if the file contents are stored successfully. + """ + minutes_sp = session.sessionpresentation_set.filter(document__type='minutes').first() + if minutes_sp: + doc = minutes_sp.document + doc.rev = '%02d' % (int(doc.rev)+1) + minutes_sp.rev = doc.rev + minutes_sp.save() + else: + ota = session.official_timeslotassignment() + sess_time = ota and ota.timeslot.time + if not sess_time: + raise SessionNotScheduledError + if session.meeting.type_id=='ietf': + name = 'minutes-%s-%s' % (session.meeting.number, + session.group.acronym) + title = 'Minutes IETF%s: %s' % (session.meeting.number, + session.group.acronym) + if not apply_to_all: + name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),) + title += ': %s' % (sess_time.strftime("%a %H:%M"),) + else: + name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M")) + title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M")) + if Document.objects.filter(name=name).exists(): + doc = Document.objects.get(name=name) + doc.rev = '%02d' % (int(doc.rev)+1) + else: + doc = Document.objects.create( + name = name, + type_id = 'minutes', + title = title, + group = session.group, + rev = '00', + ) + DocAlias.objects.create(name=doc.name).docs.add(doc) + doc.states.add(State.objects.get(type_id='minutes',slug='active')) + if session.sessionpresentation_set.filter(document=doc).exists(): + sp = session.sessionpresentation_set.get(document=doc) + sp.rev = doc.rev + sp.save() + else: + session.sessionpresentation_set.create(document=doc,rev=doc.rev) + if apply_to_all: + for other_session in get_meeting_sessions(session.meeting.number, session.group.acronym): + if other_session != session: + other_session.sessionpresentation_set.filter(document__type='minutes').delete() + other_session.sessionpresentation_set.create(document=doc,rev=doc.rev) + filename = f'{doc.name}-{doc.rev}{ext}' + doc.uploaded_filename = filename + e = NewRevisionDocEvent.objects.create( + doc=doc, + by=request.user.person, + type='new_revision', + desc=f'New revision available: {doc.rev}', + rev=doc.rev, + ) + + # The way this function builds the filename it will never trigger the file delete in handle_file_upload. + save_error = handle_upload_file( + file=file, + filename=doc.uploaded_filename, + meeting=session.meeting, + subdir='minutes', + request=request, + encoding=encoding, + ) + if save_error: + raise SaveMaterialsError(save_error) + else: + doc.save_with_history([e]) + + +def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=None): + """Accept an uploaded materials file + + This function takes a file object, a filename and a meeting object and subdir as string. + It saves the file to the appropriate directory, get_materials_path() + subdir. + If the file is a zip file, it creates a new directory in 'slides', which is the basename of the + zip file and unzips the file in the new directory. + """ + filename = Path(filename) + is_zipfile = filename.suffix == '.zip' + + path = Path(meeting.get_materials_path()) / subdir + if is_zipfile: + path = path / filename.stem + path.mkdir(parents=True, exist_ok=True) + + # agendas and minutes can only have one file instance so delete file if it already exists + if subdir in ('agenda', 'minutes'): + for f in path.glob(f'{filename.stem}.*'): + try: + f.unlink() + except FileNotFoundError: + pass # if the file is already gone, so be it + + with (path / filename).open('wb+') as destination: + if filename.suffix in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']: + file.open() + text = file.read() + if encoding: + try: + text = text.decode(encoding) + except LookupError as e: + return ( + f"Failure trying to save '{filename}': " + f"Could not identify the file encoding, got '{str(e)[:120]}'. " + f"Hint: Try to upload as UTF-8." + ) + else: + try: + text = smart_text(text) + except UnicodeDecodeError as e: + return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120]) + # Whole file sanitization; add back what's missing from a complete + # document (sanitize will remove these). + clean = sanitize_document(text) + destination.write(clean.encode('utf8')) + if request and clean != text: + messages.warning(request, + ( + f"Uploaded html content is sanitized to prevent unsafe content. " + f"Your upload {filename} was changed by the sanitization; " + f"please check the resulting content. " + )) + else: + if hasattr(file, 'chunks'): + for chunk in file.chunks(): + destination.write(chunk) + else: + destination.write(file.read()) + + # unzip zipfile + if is_zipfile: + subprocess.call(['unzip', filename], cwd=path) + + return None diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 6ef70c096..1ab18f124 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -14,7 +14,6 @@ import pytz import re import tarfile import tempfile -import markdown from calendar import timegm from collections import OrderedDict, Counter, deque, defaultdict @@ -26,7 +25,7 @@ from django import forms from django.shortcuts import render, redirect, get_object_or_404 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotFound, Http404, HttpResponseBadRequest, - JsonResponse) + JsonResponse, HttpResponseGone) from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required @@ -57,7 +56,7 @@ from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission from ietf.meeting.models import SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName -from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, +from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, TimeSlotCreateForm, TimeSlotEditForm, SessionEditForm ) from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name from ietf.meeting.helpers import get_meeting, get_ietf_meeting, get_current_ietf_meeting_num @@ -76,19 +75,20 @@ from ietf.meeting.helpers import send_interim_announcement_request from ietf.meeting.utils import finalize, sort_accept_tuple, condition_slide_order from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting -from ietf.meeting.utils import session_requested_by -from ietf.meeting.utils import current_session_status -from ietf.meeting.utils import data_for_meetings_overview +from ietf.meeting.utils import session_requested_by, SaveMaterialsError +from ietf.meeting.utils import current_session_status, get_meeting_sessions, SessionNotScheduledError +from ietf.meeting.utils import data_for_meetings_overview, handle_upload_file, save_session_minutes_revision from ietf.meeting.utils import preprocess_constraints_for_meeting_schedule_editor from ietf.meeting.utils import diff_meeting_schedules, prefetch_schedule_diff_objects from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots from ietf.meeting.utils import preprocess_meeting_important_dates from ietf.message.utils import infer_message from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName -from ietf.secr.proceedings.utils import handle_upload_file from ietf.secr.proceedings.proc_utils import (get_progress_stats, post_process, import_audio_files, create_recording) +from ietf.utils import markdown from ietf.utils.decorators import require_api_key +from ietf.utils.hedgedoc import Note, NoteError from ietf.utils.history import find_history_replacements_active_at from ietf.utils.log import assertion from ietf.utils.mail import send_mail_message, send_mail_text @@ -259,7 +259,7 @@ def materials_document(request, document, num=None, ext=None): content_type = content_type.replace('plain', 'markdown', 1) break; elif atype[0] == 'text/html': - bytes = "\n\n\n%s\n\n\n" % markdown.markdown(bytes.decode(),extensions=['extra']) + bytes = "\n\n\n%s\n\n\n" % markdown.markdown(bytes.decode()) content_type = content_type.replace('plain', 'html', 1) break; elif atype[0] == 'text/plain': @@ -2201,16 +2201,13 @@ def meeting_requests(request, num=None): {"meeting": meeting, "sessions":sessions, "groups_not_meeting": groups_not_meeting}) + def get_sessions(num, acronym): - meeting = get_meeting(num=num,type_in=None) - sessions = Session.objects.filter(meeting=meeting,group__acronym=acronym,type__in=['regular','plenary','other']) + return sorted( + get_meeting_sessions(num, acronym).with_current_status(), + key=lambda s: session_time_for_sorting(s, use_meeting_date=False) + ) - if not sessions: - sessions = Session.objects.filter(meeting=meeting,short=acronym,type__in=['regular','plenary','other']) - - sessions = sessions.with_current_status() - - return sorted(sessions, key=lambda s: session_time_for_sorting(s, use_meeting_date=False)) def session_details(request, num, acronym): meeting = get_meeting(num=num,type_in=None) @@ -2367,13 +2364,14 @@ def upload_session_bluesheets(request, session_id, num): ota = session.official_timeslotassignment() sess_time = ota and ota.timeslot.time if not sess_time: - return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain") + return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain") save_error = save_bluesheet(request, session, file, encoding=form.file_encoding[file.name]) if save_error: form.add_error(None, save_error) else: + messages.success(request, 'Successfully uploaded bluesheets.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: form = UploadBlueSheetForm() @@ -2455,62 +2453,29 @@ def upload_session_minutes(request, session_id, num): apply_to_all = session.type_id == 'regular' if show_apply_to_all_checkbox: apply_to_all = form.cleaned_data['apply_to_all'] - if minutes_sp: - doc = minutes_sp.document - doc.rev = '%02d' % (int(doc.rev)+1) - minutes_sp.rev = doc.rev - minutes_sp.save() + + # Set up the new revision + try: + save_session_minutes_revision( + session=session, + apply_to_all=apply_to_all, + file=file, + ext=ext, + encoding=form.file_encoding[file.name], + request=request, + ) + except SessionNotScheduledError: + return HttpResponseGone( + "Cannot receive uploads for an unscheduled session. Please check the session ID.", + content_type="text/plain", + ) + except SaveMaterialsError as err: + form.add_error(None, str(err)) else: - ota = session.official_timeslotassignment() - sess_time = ota and ota.timeslot.time - if not sess_time: - return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain") - if session.meeting.type_id=='ietf': - name = 'minutes-%s-%s' % (session.meeting.number, - session.group.acronym) - title = 'Minutes IETF%s: %s' % (session.meeting.number, - session.group.acronym) - if not apply_to_all: - name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),) - title += ': %s' % (sess_time.strftime("%a %H:%M"),) - else: - name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M")) - title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M")) - if Document.objects.filter(name=name).exists(): - doc = Document.objects.get(name=name) - doc.rev = '%02d' % (int(doc.rev)+1) - else: - doc = Document.objects.create( - name = name, - type_id = 'minutes', - title = title, - group = session.group, - rev = '00', - ) - DocAlias.objects.create(name=doc.name).docs.add(doc) - doc.states.add(State.objects.get(type_id='minutes',slug='active')) - if session.sessionpresentation_set.filter(document=doc).exists(): - sp = session.sessionpresentation_set.get(document=doc) - sp.rev = doc.rev - sp.save() - else: - session.sessionpresentation_set.create(document=doc,rev=doc.rev) - if apply_to_all: - for other_session in sessions: - if other_session != session: - other_session.sessionpresentation_set.filter(document__type='minutes').delete() - other_session.sessionpresentation_set.create(document=doc,rev=doc.rev) - filename = '%s-%s%s'% ( doc.name, doc.rev, ext) - doc.uploaded_filename = filename - e = NewRevisionDocEvent.objects.create(doc=doc, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev, rev=doc.rev) - # The way this function builds the filename it will never trigger the file delete in handle_file_upload. - save_error = handle_upload_file(file, filename, session.meeting, 'minutes', request=request, encoding=form.file_encoding[file.name]) - if save_error: - form.add_error(None, save_error) - else: - doc.save_with_history([e]) - return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) - else: + # no exception -- success! + messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.') + return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) + else: form = UploadMinutesForm(show_apply_to_all_checkbox) return render(request, "meeting/upload_session_minutes.html", @@ -2532,7 +2497,7 @@ def upload_session_agenda(request, session_id, num): session_number = None sessions = get_sessions(session.meeting.number,session.group.acronym) - show_apply_to_all_checkbox = len(sessions) > 1 if session.type_id == 'regular' else False + show_apply_to_all_checkbox = len(sessions) > 1 if session.type.slug == 'regular' else False if len(sessions) > 1: session_number = 1 + sessions.index(session) @@ -2543,7 +2508,7 @@ def upload_session_agenda(request, session_id, num): if form.is_valid(): file = request.FILES['file'] _, ext = os.path.splitext(file.name) - apply_to_all = session.type_id == 'regular' + apply_to_all = session.type.slug == 'regular' if show_apply_to_all_checkbox: apply_to_all = form.cleaned_data['apply_to_all'] if agenda_sp: @@ -2555,7 +2520,7 @@ def upload_session_agenda(request, session_id, num): ota = session.official_timeslotassignment() sess_time = ota and ota.timeslot.time if not sess_time: - return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain") + return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain") if session.meeting.type_id=='ietf': name = 'agenda-%s-%s' % (session.meeting.number, session.group.acronym) @@ -2603,6 +2568,7 @@ def upload_session_agenda(request, session_id, num): form.add_error(None, save_error) else: doc.save_with_history([e]) + messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: form = UploadAgendaForm(show_apply_to_all_checkbox, initial={'apply_to_all':session.type_id=='regular'}) @@ -2698,6 +2664,9 @@ def upload_session_slides(request, session_id, num, name): else: doc.save_with_history([e]) post_process(doc) + messages.success( + request, + f'Successfully uploaded slides as revision {doc.rev} of {doc.name}.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: initial = {} @@ -2764,6 +2733,7 @@ def propose_session_slides(request, session_id, num): msg.by = request.user.person msg.save() send_mail_message(request, msg) + messages.success(request, 'Successfully submitted proposed slides.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: initial = {} @@ -2787,6 +2757,7 @@ def remove_sessionpresentation(request, session_id, num, name): c = DocEvent(type="added_comment", doc=sp.document, rev=sp.document.rev, by=request.user.person) c.desc = "Removed from session: %s" % (session) c.save() + messages.success(request, f'Successfully removed {name}.') return redirect('ietf.meeting.views.session_details', num=session.meeting.number, acronym=session.group.acronym) return render(request,'meeting/remove_sessionpresentation.html', {'sp': sp }) @@ -4119,3 +4090,87 @@ def approve_proposed_slides(request, slidesubmission_id, num): 'existing_doc' : existing_doc, 'form': form, }) + + +def import_session_minutes(request, session_id, num): + """Import session minutes from the ietf.notes.org site + + A GET pulls in the markdown for a session's notes using the HedgeDoc API. An HTML preview of how + the datatracker will render the result is sent back. The confirmation form presented to the user + contains a hidden copy of the markdown source that will be submitted back if approved. + + A POST accepts the hidden source and creates a new revision of the notes. This step does *not* + retrieve the note from the HedgeDoc API again - it posts the hidden source from the form. Any + changes to the HedgeDoc site after the preview was retrieved will be ignored. We could also pull + the source again and re-display the updated preview with an explanatory message, but there will + always be a race condition. Rather than add that complication, we assume that the user previewing + the imported minutes will be aware of anyone else changing the notes and coordinate with them. + + A consequence is that the user could tamper with the hidden form and it would be accepted. This is + ok, though, because they could more simply upload whatever they want through the upload form with + the same effect so no exploit is introduced. + """ + session = get_object_or_404(Session, pk=session_id) + note = Note(session.notes_id()) + + if not session.can_manage_materials(request.user): + permission_denied(request, "You don't have permission to import minutes for this session.") + if session.is_material_submission_cutoff() and not has_role(request.user, "Secretariat"): + permission_denied(request, "The materials cutoff for this session has passed. Contact the secretariat for further action.") + + if request.method == 'POST': + form = ImportMinutesForm(request.POST) + if not form.is_valid(): + import_contents = form.data['markdown_text'] + else: + import_contents = form.cleaned_data['markdown_text'] + try: + save_session_minutes_revision( + session=session, + file=io.BytesIO(import_contents.encode('utf8')), + ext='.md', + request=request, + ) + except SessionNotScheduledError: + return HttpResponseGone( + "Cannot import minutes for an unscheduled session. Please check the session ID.", + content_type="text/plain", + ) + except SaveMaterialsError as err: + form.add_error(None, str(err)) + else: + messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.') + return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) + else: + try: + import_contents = note.get_source() + except NoteError as err: + messages.error(request, f'Could not import notes with id {note.id}: {err}.') + return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) + form = ImportMinutesForm(initial={'markdown_text': import_contents}) + + # Try to prevent pointless revision creation. Note that we do not block replacing + # a document with an identical copy in the validation above. We cannot entirely + # avoid a race condition and the likelihood/amount of damage is very low so no + # need to complicate things further. + current_minutes = session.minutes() + contents_changed = True + if current_minutes: + try: + with open(current_minutes.get_file_name()) as f: + if import_contents == Note.preprocess_source(f.read()): + contents_changed = False + messages.warning(request, 'This document is identical to the current revision, no need to import.') + except FileNotFoundError: + pass # allow import if the file is missing + + return render( + request, + 'meeting/import_minutes.html', + { + 'form': form, + 'note': note, + 'session': session, + 'contents_changed': contents_changed, + }, + ) diff --git a/ietf/meeting/views_proceedings.py b/ietf/meeting/views_proceedings.py index cfb38e5ab..f589b9c95 100644 --- a/ietf/meeting/views_proceedings.py +++ b/ietf/meeting/views_proceedings.py @@ -14,7 +14,7 @@ from ietf.meeting.forms import FileUploadForm from ietf.meeting.models import Meeting, MeetingHost from ietf.meeting.helpers import get_meeting from ietf.name.models import ProceedingsMaterialTypeName -from ietf.secr.proceedings.utils import handle_upload_file +from ietf.meeting.utils import handle_upload_file from ietf.utils.text import xslugify class UploadProceedingsMaterialForm(FileUploadForm): diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py index 6582cb81e..59f8eedb6 100644 --- a/ietf/secr/meetings/views.py +++ b/ietf/secr/meetings/views.py @@ -20,14 +20,13 @@ from ietf.utils.mail import send_mail from ietf.meeting.forms import duration_string from ietf.meeting.helpers import get_meeting, make_materials_directories, populate_important_dates from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent -from ietf.meeting.utils import add_event_info_to_session_qs +from ietf.meeting.utils import add_event_info_to_session_qs, handle_upload_file from ietf.name.models import SessionStatusName from ietf.group.models import Group, GroupEvent from ietf.secr.meetings.blue_sheets import create_blue_sheets from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm, MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm, UploadBlueSheetForm, MeetingRoomOptionsForm ) -from ietf.secr.proceedings.utils import handle_upload_file from ietf.secr.sreq.views import get_initial_session from ietf.secr.utils.meeting import get_session, get_timeslot from ietf.mailtrigger.utils import gather_address_lists diff --git a/ietf/secr/proceedings/utils.py b/ietf/secr/proceedings/utils.py deleted file mode 100644 index 73f9dda24..000000000 --- a/ietf/secr/proceedings/utils.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved - -import glob -import io -import os - -from django.conf import settings -from django.contrib import messages -from django.utils.encoding import smart_text - -import debug # pyflakes:ignore - -from ietf.utils.html import sanitize_document - -def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None): - ''' - This function takes a file object, a filename and a meeting object and subdir as string. - It saves the file to the appropriate directory, get_materials_path() + subdir. - If the file is a zip file, it creates a new directory in 'slides', which is the basename of the - zip file and unzips the file in the new directory. - ''' - base, extension = os.path.splitext(filename) - - if extension == '.zip': - path = os.path.join(meeting.get_materials_path(),subdir,base) - if not os.path.exists(path): - os.mkdir(path) - else: - path = os.path.join(meeting.get_materials_path(),subdir) - if not os.path.exists(path): - os.makedirs(path) - - # agendas and minutes can only have one file instance so delete file if it already exists - if subdir in ('agenda','minutes'): - old_files = glob.glob(os.path.join(path,base) + '.*') - for f in old_files: - os.remove(f) - - destination = io.open(os.path.join(path,filename), 'wb+') - if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']: - file.open() - text = file.read() - if encoding: - try: - text = text.decode(encoding) - except LookupError as e: - return "Failure trying to save '%s': Could not identify the file encoding, got '%s'. Hint: Try to upload as UTF-8." % (filename, str(e)[:120]) - else: - try: - text = smart_text(text) - except UnicodeDecodeError as e: - return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120]) - # Whole file sanitization; add back what's missing from a complete - # document (sanitize will remove these). - clean = sanitize_document(text) - destination.write(clean.encode('utf8')) - if request and clean != text: - messages.warning(request, "Uploaded html content is sanitized to prevent unsafe content. " - "Your upload %s was changed by the sanitization; please check the " - "resulting content. " % (filename, )) - else: - if hasattr(file, 'chunks'): - for chunk in file.chunks(): - destination.write(chunk) - else: - destination.write(file.read()) - destination.close() - - # unzip zipfile - if extension == '.zip': - os.chdir(path) - os.system('unzip %s' % filename) - - return None diff --git a/ietf/settings.py b/ietf/settings.py index a6c4c8188..24058eafa 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -148,6 +148,7 @@ IETF_ID_URL = IETF_HOST_URL + 'id/' # currently unused IETF_ID_ARCHIVE_URL = IETF_HOST_URL + 'archive/id/' IETF_AUDIO_URL = IETF_HOST_URL + 'audio/' +IETF_NOTES_URL = 'https://notes.ietf.org/' # HedgeDoc base URL # Absolute path to the directory static files should be collected to. # Example: "/var/www/example.com/static/" @@ -996,6 +997,7 @@ DOT_BINARY = '/usr/bin/dot' UNFLATTEN_BINARY= '/usr/bin/unflatten' RSYNC_BINARY = '/usr/bin/rsync' YANGLINT_BINARY = '/usr/bin/yanglint' +DE_GFM_BINARY = '/usr/bin/de-gfm.ruby2.5' # Account settings DAYS_TO_EXPIRE_REGISTRATION_LINK = 3 diff --git a/ietf/templates/meeting/import_minutes.html b/ietf/templates/meeting/import_minutes.html new file mode 100644 index 000000000..68ecb5d88 --- /dev/null +++ b/ietf/templates/meeting/import_minutes.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% load bootstrap3 origin %} + +{% block morecss %} + #preview { width: 100%; height: 60vh; border: solid 2px; } +{% endblock %} + +{% block title %} + Import Preview: {% firstof note.get_title note.id %} +{% endblock %} + +{% block content %} + {% origin %} +

Import Preview: {% firstof note.get_title note.id %}

+
+
+

Last updated: {% firstof note.get_update_time "unknown" %}

+

View on notes.ietf.org

+ +
+
+
+
+
+ {% csrf_token %} + {% bootstrap_form form %} + {% buttons %} + + Back + {% endbuttons %} +
+
+
+{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/interim_session_buttons.html b/ietf/templates/meeting/interim_session_buttons.html index 0a406819c..ce4866736 100644 --- a/ietf/templates/meeting/interim_session_buttons.html +++ b/ietf/templates/meeting/interim_session_buttons.html @@ -16,11 +16,7 @@ {% endif %} {% if use_codimd %} - {% if item.slot_type.slug == 'plenary' %} - - {% else %} - - {% endif %} + {% endif %} {# show stream buttons up till end of session, then show archive buttons #} diff --git a/ietf/templates/meeting/session_buttons_include.html b/ietf/templates/meeting/session_buttons_include.html index 30245b514..e2f17b142 100644 --- a/ietf/templates/meeting/session_buttons_include.html +++ b/ietf/templates/meeting/session_buttons_include.html @@ -22,11 +22,7 @@ {% if use_codimd %} - {% if timeslot.type.slug == 'plenary' %} - - {% else %} - - {% endif %} + {% endif %} {# show stream buttons up till end of session, then show archive buttons #} diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html index f499c96bd..eb08c36ec 100644 --- a/ietf/templates/meeting/session_details_panel.html +++ b/ietf/templates/meeting/session_details_panel.html @@ -74,6 +74,9 @@ {% endif %} {% if pres.document.type.slug != 'bluesheets' or user|has_role:"Secretariat" or meeting.type.slug == 'interim' and can_manage_materials %} Upload Revision + {% if pres.document.type.slug == 'minutes' %} + Import from notes.ietf.org + {% endif %} {% endif %} {% endif %} diff --git a/ietf/utils/hedgedoc.py b/ietf/utils/hedgedoc.py new file mode 100644 index 000000000..f02d1ffd3 --- /dev/null +++ b/ietf/utils/hedgedoc.py @@ -0,0 +1,117 @@ +# Copyright The IETF Trust 2021, All Rights Reserved +# -*- coding: utf-8 -*- +"""HedgeDoc API utilities""" +import json +import requests +import subprocess +import debug # pyflakes: ignore + +from urllib.parse import urljoin +from django.conf import settings + +from ietf.utils.markdown import markdown + + +class Note: + base_url = settings.IETF_NOTES_URL + + def __init__(self, id): + self.id = id + self.url = urljoin(self.base_url, self.id) # URL on notes site + self._metadata = None + self._preview = None + self._source = None + + @classmethod + def preprocess_source(cls, raw_source): + """Perform preprocessing on raw source input + + Guaranteed to process input in the same way that the Note class processes + markdown source pulled from the notes site. + """ + return de_gfm(raw_source) + + def get_source(self): + """Retrieve markdown source from hedgedoc + + Converts line breaks from GitHub Flavored Markdown (GFM) style to + to traditional markdown. + """ + if self._source is None: + try: + r = requests.get(urljoin(self.base_url, f'{self.id}/download'), allow_redirects=True) + except requests.RequestException: + raise ServerNoteError + if r.status_code != 200: + raise NoteNotFound + self._source = self.preprocess_source(r.text) + return self._source + + def get_preview(self): + if self._preview is None: + self._preview = markdown(self.get_source()) + return self._preview + + def get_title(self): + try: + metadata = self._retrieve_metadata() + except NoteError: + metadata = {} # don't let an error retrieving the title prevent retrieval + return metadata.get('title', None) + + def get_update_time(self): + try: + metadata = self._retrieve_metadata() + except NoteError: + metadata = {} # don't let an error retrieving the update timestamp prevent retrieval + return metadata.get('updatetime', None) + + def _retrieve_metadata(self): + if self._metadata is None: + try: + r = requests.get(urljoin(self.base_url, f'{self.id}/info'), allow_redirects=True) + except requests.RequestException: + raise ServerNoteError + if r.status_code != 200: + raise NoteNotFound + try: + self._metadata = json.loads(r.content) + except json.JSONDecodeError: + raise InvalidNote + return self._metadata + + +def de_gfm(source: str): + """Convert GFM line breaks to standard Markdown + + Calls de-gfm from the kramdown-rfc2629 gem. + """ + result = subprocess.run( + [settings.DE_GFM_BINARY,], + stdout=subprocess.PIPE, # post-Python 3.7, this can be replaced with capture_output=True + input=source, + encoding='utf8', + check=True, + ) + return result.stdout + + +class NoteError(Exception): + """Base class for exceptions in this module""" + default_message = 'A note-related error occurred' + + def __init__(self, *args, **kwargs): + if not args: + args = (self.default_message, ) + super().__init__(*args) + + +class ServerNoteError(NoteError): + default_message = 'Could not reach the notes server' + +class NoteNotFound(NoteError): + default_message = 'Note did not exist or could not be loaded' + + +class InvalidNote(NoteError): + default_message = 'Note data invalid' \ No newline at end of file diff --git a/ietf/utils/markdown.py b/ietf/utils/markdown.py new file mode 100644 index 000000000..2ac562f26 --- /dev/null +++ b/ietf/utils/markdown.py @@ -0,0 +1,20 @@ +# Copyright The IETF Trust 2021, All Rights Reserved +# -*- coding: utf-8 -*- +"""Markdown wrapper + +Use this instead of importing markdown directly to guarantee consistent extensions / options through +the datatracker. +""" +import bleach +import markdown as python_markdown + +from django.utils.safestring import mark_safe +from markdown.extensions.extra import ExtraExtension + +ALLOWED_TAGS = bleach.ALLOWED_TAGS + ['p', 'h1', 'h2', 'h3', 'h4', 'br'] + +def markdown(text): + return mark_safe(bleach.clean( + python_markdown.markdown(text, extensions=[ExtraExtension()]), + tags=ALLOWED_TAGS, + )) diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 1098a0acb..abde0de74 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -38,6 +38,7 @@ import os import re import email import html5lib +import requests_mock import shutil import sys @@ -151,6 +152,7 @@ class ReverseLazyTest(django.test.TestCase): response = self.client.get('/ipr/update/') self.assertRedirects(response, "/ipr/", status_code=301) + class TestCase(django.test.TestCase): """IETF TestCase class @@ -158,6 +160,7 @@ class TestCase(django.test.TestCase): * asserts for html5 validation. * tempdir() convenience method * setUp() and tearDown() that override settings paths with temp directories + * mocking the requests library to prevent dependencies on the outside network The setUp() and tearDown() methods create / remove temporary paths and override Django's settings with the temp dir names. Subclasses of this class must @@ -165,6 +168,12 @@ class TestCase(django.test.TestCase): anew for each test to avoid risk of cross-talk between test cases. Overriding the settings_temp_path_overrides class value will modify which path settings are replaced with temp test dirs. + + Uses requests-mock to prevent the requests library from making requests to outside + resources. The requests-mock library allows nested mocks, so individual tests can + ignore this. Note that the mock set up by this class will intercept any requests + not handled by a test's inner mock - even if the latter is created with + real_http=True. """ # These settings will be overridden with empty temporary directories settings_temp_path_overrides = [ @@ -257,10 +266,14 @@ class TestCase(django.test.TestCase): def __str__(self): return u"%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) - def setUp(self): - # Replace settings paths with temporary directories. super().setUp() + + # Prevent the requests library from making live requests during tests + self.requests_mock = requests_mock.Mocker() + self.requests_mock.start() + + # Replace settings paths with temporary directories. self._ietf_temp_dirs = {} # trashed during tearDown, DO NOT put paths you care about in this for setting in self.settings_temp_path_overrides: self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting)) @@ -271,4 +284,5 @@ class TestCase(django.test.TestCase): self._ietf_saved_context.disable() for dir in self._ietf_temp_dirs.values(): shutil.rmtree(dir) + self.requests_mock.stop() super().tearDown() diff --git a/ietf/utils/tests_hedgedoc.py b/ietf/utils/tests_hedgedoc.py new file mode 100644 index 000000000..1450c90df --- /dev/null +++ b/ietf/utils/tests_hedgedoc.py @@ -0,0 +1,45 @@ +# Copyright The IETF Trust 2021, All Rights Reserved +# -*- coding: utf-8 -*- +"""HedgeDoc API utilities tests""" +import requests_mock + +from ietf.utils.tests import TestCase +from ietf.utils.hedgedoc import Note + + +class NoteTests(TestCase): + SAMPLE_MARKDOWN = ''.join(( + '# Standard Markdown\n', + 'This is a small sample of markdown text. It uses GFM-style line breaks.\n', + '\n', + 'This is a second paragraph.\n', + 'It has line breaks in GFM style.\n', + 'And also some standard line breaks. \n', + 'That is all.\n', + )) + SAMPLE_MARKDOWN_OUTPUT = ''.join(( + '# Standard Markdown {#standard-markdown}\n', + '\n', + 'This is a small sample of markdown text. It uses GFM-style line breaks.\n', + '\n', + 'This is a second paragraph. \n', + 'It has line breaks in GFM style. \n', + 'And also some standard line breaks. \n', + 'That is all.\n', + '\n', + )) + + def test_retrieves_note(self): + with requests_mock.Mocker() as mock: + mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN) + n = Note('my_id') + result = n.get_source() + self.assertEqual(result, self.SAMPLE_MARKDOWN_OUTPUT) + + def test_uses_preprocess_class_method(self): + """Imported text should be processed by the preprocess_source class method""" + with requests_mock.Mocker() as mock: + mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN) + n = Note('my_id') + result = n.get_source() + self.assertEqual(result, Note.preprocess_source(self.SAMPLE_MARKDOWN))