Add ability to import session minutes from notes.ietf.org. Mock out calls to the requests library in tests. Call markdown library through a util method. Fixes #3489. Commit ready for merge.

- Legacy-Id: 19763

ietf/doc/views_bofreq.py

@@ -3,7 +3,6 @@
 import debug    # pyflakes:ignore
 
 import io 
-import markdown
 
 from django import forms
 from django.contrib.auth.decorators import login_required
@@ -20,6 +19,7 @@ from ietf.doc.utils import add_state_change_event
 from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
 from ietf.ietfauth.utils import has_role, role_required
 from ietf.person.fields import SearchablePersonsField
+from ietf.utils import markdown
 from ietf.utils.response import permission_denied
 from ietf.utils.text import xslugify
 from ietf.utils.textupload import get_cleaned_text_file_content
@@ -64,7 +64,7 @@ class BofreqUploadForm(forms.Form):
             if require_field("bofreq_file"):
                 content = get_cleaned_text_file_content(self.cleaned_data["bofreq_file"])
         try:
-            _ = markdown.markdown(content, extensions=['extra'])
+            _ = markdown.markdown(content)
         except Exception as e:
            raise forms.ValidationError(f'Markdown processing failed: {e}')
 

ietf/doc/views_doc.py

@@ -40,7 +40,6 @@ import io
 import json
 import os
 import re
-import markdown
 
 from urllib.parse import quote
 
@@ -80,7 +79,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import markup_txt, log
+from ietf.utils import markup_txt, log, markdown
 from ietf.utils.draft import Draft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
@@ -550,7 +549,7 @@ def document_main(request, name, rev=None):
                                        ))
 
     if doc.type_id == "bofreq":
-        content = markdown.markdown(doc.text_or_error(),extensions=['extra'])
+        content = markdown.markdown(doc.text_or_error())
         editors = bofreq_editors(doc)
         responsible = bofreq_responsible(doc)
         can_manage = has_role(request.user,['Secretariat', 'Area Director', 'IAB'])
@@ -661,7 +660,7 @@ def document_main(request, name, rev=None):
                 content = doc.text_or_error()
                 t = "plain text"
             elif extension == ".md":
-                content = markdown.markdown(doc.text_or_error(), extensions=['extra'])
+                content = markdown.markdown(doc.text_or_error())
                 content_is_html = True
                 t = "markdown"
             other_types.append((t, url))

ietf/group/views.py

@@ -38,7 +38,6 @@ import copy
 import datetime
 import itertools
 import io
-import markdown
 import math
 import os
 import re
@@ -121,7 +120,7 @@ from ietf.settings import MAILING_LIST_INFO_URL
 from ietf.utils.pipe import pipe
 from ietf.utils.response import permission_denied
 from ietf.utils.text import strip_suffix
-
+from ietf.utils import markdown
 
 
 # --- Helpers ----------------------------------------------------------
@@ -581,7 +580,7 @@ def group_about_rendertest(request, acronym, group_type=None):
     if group.charter:
         charter = get_charter_text(group)
     try:
-        rendered = markdown.markdown(charter, extensions=['extra'])
+        rendered = markdown.markdown(charter)
     except Exception as e:
         rendered = f'Markdown rendering failed: {e}'
     return render(request, 'group/group_about_rendertest.html', {'group':group, 'charter':charter, 'rendered':rendered})

ietf/meeting/forms.py

@@ -436,6 +436,10 @@ class UploadSlidesForm(ApplyToAllFileUploadForm):
         return title
 
 
+class ImportMinutesForm(forms.Form):
+    markdown_text = forms.CharField(strip=False, widget=forms.HiddenInput)
+
+
 class RequestMinutesForm(forms.Form):
     to = MultiEmailField()
     cc = MultiEmailField(required=False)

ietf/meeting/models.py

@@ -14,6 +14,7 @@ import string
 
 from collections import namedtuple
 from pathlib import Path
+from urllib.parse import urljoin
 
 import debug                            # pyflakes:ignore
 
@@ -1260,6 +1261,13 @@ class Session(models.Model):
         else:
             return self.group.acronym
 
+    def notes_id(self):
+        note_id_fragment = 'plenary' if self.type.slug == 'plenary' else self.group.acronym
+        return f'notes-ietf-{self.meeting.number}-{note_id_fragment}'
+
+    def notes_url(self):
+        return urljoin(settings.IETF_NOTES_URL, self.notes_id())
+
 class SchedulingEvent(models.Model):
     session = ForeignKey(Session)
     time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened")

ietf/meeting/tests_views.py

@@ -8,6 +8,8 @@ import random
 import re
 import shutil
 import pytz
+import requests.exceptions
+import requests_mock
 
 from unittest import skipIf
 from mock import patch, PropertyMock
@@ -19,7 +21,6 @@ from urllib.parse import urlparse, urlsplit
 from PIL import Image
 from pathlib import Path
 
-
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
@@ -5405,12 +5406,16 @@ class IphoneAppJsonTests(TestCase):
             self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
 
 class FinalizeProceedingsTests(TestCase):
-    @patch('urllib.request.urlopen')
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
-    def test_finalize_proceedings(self, mock_urlopen):
+    @requests_mock.Mocker()
-        mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
+    def test_finalize_proceedings(self, mock):
         make_meeting_test_data()
         meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last()
         meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None)
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
 
         url = urlreverse('ietf.meeting.views.finalize_proceedings',kwargs={'num':meeting.number})
         login_testing_unauthorized(self,"secretary",url)
@@ -5605,8 +5610,10 @@ class MaterialsTests(TestCase):
             self.assertEqual(doc.rev,'02')
 
             # Verify that we don't have dead links
-            url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
             top = '/meeting/%s/' % session.meeting.number
+            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
             self.crawl_materials(url=url, top=top)
 
     def test_upload_minutes_agenda_unscheduled(self):
@@ -5653,8 +5660,10 @@ class MaterialsTests(TestCase):
             self.assertEqual(doc.rev,'00')
 
             # Verify that we don't have dead links
-            url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
+            url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
             top = '/meeting/%s/' % session.meeting.number
+            self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
+            self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
             self.crawl_materials(url=url, top=top)
 
     def test_upload_slides(self):
@@ -5928,6 +5937,151 @@ class MaterialsTests(TestCase):
         self.assertIn('third version', contents)
 
 
+@override_settings(IETF_NOTES_URL='https://notes.ietf.org/')
+class ImportNotesTests(TestCase):
+    settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
+
+    def setUp(self):
+        super().setUp()
+        self.session = SessionFactory(meeting__type_id='ietf')
+        self.meeting = self.session.meeting
+
+    def test_retrieves_note(self):
+        """Can import and preview a note from notes.ietf.org"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'markdown text')
+
+    def test_retrieves_with_broken_metadata(self):
+        """Can import and preview a note even if it has a metadata problem"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', text='this is not valid json {]')
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'markdown text')
+
+    def test_redirects_on_success(self):
+        """Redirects to session details page after import"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'markdown text'})
+        self.assertRedirects(
+            r,
+            urlreverse(
+                'ietf.meeting.views.session_details',
+                kwargs={
+                    'num': self.meeting.number,
+                    'acronym': self.session.group.acronym,
+                },
+            ),
+        )
+
+    def test_imports_previewed_text(self):
+        """Import text that was shown as preview even if notes site is updated"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='updated markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.post(url, {'markdown_text': 'original markdown text'})
+        self.assertEqual(r.status_code, 302)
+        minutes_path = Path(self.meeting.get_materials_path()) / 'minutes'
+        with (minutes_path / self.session.minutes().uploaded_filename).open() as f:
+            self.assertEqual(f.read(), 'original markdown text')
+
+    def test_refuses_identical_import(self):
+        """Should not be able to import text identical to the current revision"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'original markdown text'})  # create a rev
+        self.assertEqual(r.status_code, 302)
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)  # try to import the same text
+            self.assertContains(r, "This document is identical", status_code=200)
+            q = PyQuery(r.content)
+            self.assertEqual(len(q('button:disabled[type="submit"]')), 1)
+            self.assertEqual(len(q('button:not(:disabled)[type="submit"]')), 0)
+
+    def test_handles_missing_previous_revision_file(self):
+        """Should still allow import if the file for the previous revision is missing"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        r = self.client.post(url, {'markdown_text': 'original markdown text'})  # create a rev
+        # remove the file uploaded for the first rev
+        minutes_docs = self.session.sessionpresentation_set.filter(document__type='minutes')
+        self.assertEqual(minutes_docs.count(), 1)
+        Path(minutes_docs.first().document.get_file_name()).unlink()
+
+        self.assertEqual(r.status_code, 302)
+        with requests_mock.Mocker() as mock:
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
+            mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
+                     text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
+            r = self.client.get(url)
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            iframe = q('iframe#preview')
+            self.assertEqual('<p>original markdown text</p>', iframe.attr('srcdoc'))
+            markdown_text_input = q('form #id_markdown_text')
+            self.assertEqual(markdown_text_input.val(), 'original markdown text')
+
+    def test_handles_note_does_not_exist(self):
+        """Should not try to import a note that does not exist"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+
+        self.client.login(username='secretary', password='secretary+password')
+        with requests_mock.Mocker() as mock:
+            mock.get(requests_mock.ANY, status_code=404)
+            r = self.client.get(url, follow=True)
+        self.assertContains(r, 'Could not import', status_code=200)
+
+    def test_handles_notes_server_failure(self):
+        """Problems communicating with the notes server should be handled gracefully"""
+        url = urlreverse('ietf.meeting.views.import_session_minutes',
+                         kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
+        self.client.login(username='secretary', password='secretary+password')
+
+        with requests_mock.Mocker() as mock:
+            mock.get(re.compile(r'.+/download'), exc=requests.exceptions.ConnectTimeout)
+            mock.get(re.compile(r'.+//info'), text='{}')
+            r = self.client.get(url, follow=True)
+        self.assertContains(r, 'Could not reach the notes server', status_code=200)
+
+
 class SessionTests(TestCase):
 
     def test_meeting_requests(self):
@@ -6911,12 +7065,15 @@ class ProceedingsTests(BaseMeetingTestCase):
             0,
         )
 
-    @patch('ietf.meeting.utils.requests.get')
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
-    def test_proceedings_attendees(self, mockobj):
+    @requests_mock.Mocker()
-        mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
+    def test_proceedings_attendees(self, mock):
-        mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
         finalize(meeting)
         url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':97})
         response = self.client.get(url)
@@ -6924,14 +7081,18 @@ class ProceedingsTests(BaseMeetingTestCase):
         q = PyQuery(response.content)
         self.assertEqual(1,len(q("#id_attendees tbody tr")))
 
-    @patch('urllib.request.urlopen')
+    @override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
-    def test_proceedings_overview(self, mock_urlopen):
+    @requests_mock.Mocker()
+    def test_proceedings_overview(self, mock):
         '''Test proceedings IETF Overview page.
         Note: old meetings aren't supported so need to add a new meeting then test.
         '''
-        mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
+        mock.get(
+            settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
+            text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
+        )
         finalize(meeting)
         url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':97})
         response = self.client.get(url)

ietf/meeting/urls.py

@@ -13,6 +13,7 @@ safe_for_all_meeting_types = [
     url(r'^session/(?P<session_id>\d+)/bluesheets$', views.upload_session_bluesheets),
     url(r'^session/(?P<session_id>\d+)/minutes$', views.upload_session_minutes),
     url(r'^session/(?P<session_id>\d+)/agenda$', views.upload_session_agenda),
+    url(r'^session/(?P<session_id>\d+)/import/minutes$', views.import_session_minutes),
     url(r'^session/(?P<session_id>\d+)/propose_slides$', views.propose_session_slides),
     url(r'^session/(?P<session_id>\d+)/slides(?:/%(name)s)?$' % settings.URL_REGEXPS, views.upload_session_slides),
     url(r'^session/(?P<session_id>\d+)/add_to_session$', views.ajax_add_slides_to_session),

ietf/meeting/utils.py

@@ -1,17 +1,19 @@
 # Copyright The IETF Trust 2016-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
-
-
 import datetime
 import itertools
 import re
 import requests
+import subprocess
 
 from collections import defaultdict
+from pathlib import Path
 from urllib.error import HTTPError
 
 from django.conf import settings
+from django.contrib import messages
 from django.template.loader import render_to_string
+from django.utils.encoding import smart_text
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
@@ -19,11 +21,14 @@ import debug                            # pyflakes:ignore
 
 from ietf.dbtemplate.models import DBTemplate
 from ietf.meeting.models import Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment
+from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
 from ietf.name.models import SessionStatusName, ConstraintName
 from ietf.person.models import Person
 from ietf.secr.proceedings.proc_utils import import_audio_files
+from ietf.utils.html import sanitize_document
+
 
 def session_time_for_sorting(session, use_meeting_date):
     official_timeslot = TimeSlot.objects.filter(sessionassignments__session=session, sessionassignments__schedule__in=[session.meeting.schedule, session.meeting.schedule.base if session.meeting.schedule else None]).first()
@@ -544,3 +549,179 @@ def preprocess_meeting_important_dates(meetings):
         for d in m.important_dates:
             d.midnight_cutoff = "UTC 23:59" in d.name.name
     
+
+def get_meeting_sessions(num, acronym):
+    types = ['regular','plenary','other']
+    sessions = Session.objects.filter(
+        meeting__number=num,
+        group__acronym=acronym,
+        type__in=types,
+    )
+    if not sessions:
+        sessions = Session.objects.filter(
+            meeting__number=num,
+            short=acronym,
+            type__in=types,
+        )
+    return sessions
+
+
+class SessionNotScheduledError(Exception):
+    """Indicates failure because operation requires a scheduled session"""
+    pass
+
+
+class SaveMaterialsError(Exception):
+    """Indicates failure saving session materials"""
+    pass
+
+
+def save_session_minutes_revision(session, file, ext, request, encoding=None, apply_to_all=False):
+    """Creates or updates session minutes records
+
+    This updates the database models to reflect a new version. It does not handle
+    storing the new file contents, that should be handled via handle_upload_file()
+    or similar.
+
+    If the session does not already have minutes, it must be a scheduled
+    session. If not, SessionNotScheduledError will be raised.
+
+    Returns (Document, [DocEvents]), which should be passed to doc.save_with_history()
+    if the file contents are stored successfully.
+    """
+    minutes_sp = session.sessionpresentation_set.filter(document__type='minutes').first()
+    if minutes_sp:
+        doc = minutes_sp.document
+        doc.rev = '%02d' % (int(doc.rev)+1)
+        minutes_sp.rev = doc.rev
+        minutes_sp.save()
+    else:
+        ota = session.official_timeslotassignment()
+        sess_time = ota and ota.timeslot.time
+        if not sess_time:
+            raise SessionNotScheduledError
+        if session.meeting.type_id=='ietf':
+            name = 'minutes-%s-%s' % (session.meeting.number,
+                                      session.group.acronym)
+            title = 'Minutes IETF%s: %s' % (session.meeting.number,
+                                            session.group.acronym)
+            if not apply_to_all:
+                name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),)
+                title += ': %s' % (sess_time.strftime("%a %H:%M"),)
+        else:
+            name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M"))
+            title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M"))
+        if Document.objects.filter(name=name).exists():
+            doc = Document.objects.get(name=name)
+            doc.rev = '%02d' % (int(doc.rev)+1)
+        else:
+            doc = Document.objects.create(
+                name = name,
+                type_id = 'minutes',
+                title = title,
+                group = session.group,
+                rev = '00',
+            )
+            DocAlias.objects.create(name=doc.name).docs.add(doc)
+        doc.states.add(State.objects.get(type_id='minutes',slug='active'))
+        if session.sessionpresentation_set.filter(document=doc).exists():
+            sp = session.sessionpresentation_set.get(document=doc)
+            sp.rev = doc.rev
+            sp.save()
+        else:
+            session.sessionpresentation_set.create(document=doc,rev=doc.rev)
+    if apply_to_all:
+        for other_session in get_meeting_sessions(session.meeting.number, session.group.acronym):
+            if other_session != session:
+                other_session.sessionpresentation_set.filter(document__type='minutes').delete()
+                other_session.sessionpresentation_set.create(document=doc,rev=doc.rev)
+    filename = f'{doc.name}-{doc.rev}{ext}'
+    doc.uploaded_filename = filename
+    e = NewRevisionDocEvent.objects.create(
+        doc=doc,
+        by=request.user.person,
+        type='new_revision',
+        desc=f'New revision available: {doc.rev}',
+        rev=doc.rev,
+    )
+
+    # The way this function builds the filename it will never trigger the file delete in handle_file_upload.
+    save_error = handle_upload_file(
+        file=file,
+        filename=doc.uploaded_filename,
+        meeting=session.meeting,
+        subdir='minutes',
+        request=request,
+        encoding=encoding,
+    )
+    if save_error:
+        raise SaveMaterialsError(save_error)
+    else:
+        doc.save_with_history([e])
+
+
+def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=None):
+    """Accept an uploaded materials file
+
+    This function takes a file object, a filename and a meeting object and subdir as string.
+    It saves the file to the appropriate directory, get_materials_path() + subdir.
+    If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
+    zip file and unzips the file in the new directory.
+    """
+    filename = Path(filename)
+    is_zipfile = filename.suffix == '.zip'
+
+    path = Path(meeting.get_materials_path()) / subdir
+    if is_zipfile:
+        path = path / filename.stem
+    path.mkdir(parents=True, exist_ok=True)
+
+    # agendas and minutes can only have one file instance so delete file if it already exists
+    if subdir in ('agenda', 'minutes'):
+        for f in path.glob(f'{filename.stem}.*'):
+            try:
+                f.unlink()
+            except FileNotFoundError:
+                pass  # if the file is already gone, so be it
+
+    with (path / filename).open('wb+') as destination:
+        if filename.suffix in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
+            file.open()
+            text = file.read()
+            if encoding:
+                try:
+                    text = text.decode(encoding)
+                except LookupError as e:
+                    return (
+                        f"Failure trying to save '{filename}': "
+                        f"Could not identify the file encoding, got '{str(e)[:120]}'. "
+                        f"Hint: Try to upload as UTF-8."
+                    )
+            else:
+                try:
+                    text = smart_text(text)
+                except UnicodeDecodeError as e:
+                    return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
+            # Whole file sanitization; add back what's missing from a complete
+            # document (sanitize will remove these).
+            clean = sanitize_document(text)
+            destination.write(clean.encode('utf8'))
+            if request and clean != text:
+                messages.warning(request,
+                                 (
+                                     f"Uploaded html content is sanitized to prevent unsafe content. "
+                                     f"Your upload {filename} was changed by the sanitization; "
+                                     f"please check the resulting content.  "
+                                 ))
+        else:
+            if hasattr(file, 'chunks'):
+                for chunk in file.chunks():
+                    destination.write(chunk)
+            else:
+                destination.write(file.read())
+
+    # unzip zipfile
+    if is_zipfile:
+        subprocess.call(['unzip', filename], cwd=path)
+
+    return None

ietf/meeting/views.py

@@ -14,7 +14,6 @@ import pytz
 import re
 import tarfile
 import tempfile
-import markdown
 
 from calendar import timegm
 from collections import OrderedDict, Counter, deque, defaultdict
@@ -26,7 +25,7 @@ from django import forms
 from django.shortcuts import render, redirect, get_object_or_404
 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden,
                          HttpResponseNotFound, Http404, HttpResponseBadRequest,
-                         JsonResponse)
+                         JsonResponse, HttpResponseGone)
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
@@ -57,7 +56,7 @@ from ietf.ietfauth.utils import role_required, has_role, user_is_person
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission
 from ietf.meeting.models import SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
-from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm,
+from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm,
                                  TimeSlotCreateForm, TimeSlotEditForm, SessionEditForm )
 from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name
 from ietf.meeting.helpers import get_meeting, get_ietf_meeting, get_current_ietf_meeting_num
@@ -76,19 +75,20 @@ from ietf.meeting.helpers import send_interim_announcement_request
 from ietf.meeting.utils import finalize, sort_accept_tuple, condition_slide_order
 from ietf.meeting.utils import add_event_info_to_session_qs
 from ietf.meeting.utils import session_time_for_sorting
-from ietf.meeting.utils import session_requested_by
+from ietf.meeting.utils import session_requested_by, SaveMaterialsError
-from ietf.meeting.utils import current_session_status
+from ietf.meeting.utils import current_session_status, get_meeting_sessions, SessionNotScheduledError
-from ietf.meeting.utils import data_for_meetings_overview
+from ietf.meeting.utils import data_for_meetings_overview, handle_upload_file, save_session_minutes_revision
 from ietf.meeting.utils import preprocess_constraints_for_meeting_schedule_editor
 from ietf.meeting.utils import diff_meeting_schedules, prefetch_schedule_diff_objects
 from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots
 from ietf.meeting.utils import preprocess_meeting_important_dates
 from ietf.message.utils import infer_message
 from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
-from ietf.secr.proceedings.utils import handle_upload_file
 from ietf.secr.proceedings.proc_utils import (get_progress_stats, post_process, import_audio_files,
     create_recording)
+from ietf.utils import markdown
 from ietf.utils.decorators import require_api_key
+from ietf.utils.hedgedoc import Note, NoteError
 from ietf.utils.history import find_history_replacements_active_at
 from ietf.utils.log import assertion
 from ietf.utils.mail import send_mail_message, send_mail_text
@@ -259,7 +259,7 @@ def materials_document(request, document, num=None, ext=None):
                     content_type = content_type.replace('plain', 'markdown', 1)
                     break;
                 elif atype[0] == 'text/html':
-                    bytes = "<html>\n<head></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode(),extensions=['extra'])
+                    bytes = "<html>\n<head></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode())
                     content_type = content_type.replace('plain', 'html', 1)
                     break;
                 elif atype[0] == 'text/plain':
@@ -2201,16 +2201,13 @@ def meeting_requests(request, num=None):
         {"meeting": meeting, "sessions":sessions,
          "groups_not_meeting": groups_not_meeting})
 
+
 def get_sessions(num, acronym):
-    meeting = get_meeting(num=num,type_in=None)
+    return sorted(
-    sessions = Session.objects.filter(meeting=meeting,group__acronym=acronym,type__in=['regular','plenary','other'])
+        get_meeting_sessions(num, acronym).with_current_status(),
+        key=lambda s: session_time_for_sorting(s, use_meeting_date=False)
+    )
 
-    if not sessions:
-        sessions = Session.objects.filter(meeting=meeting,short=acronym,type__in=['regular','plenary','other']) 
-
-    sessions = sessions.with_current_status()
-
-    return sorted(sessions, key=lambda s: session_time_for_sorting(s, use_meeting_date=False))
 
 def session_details(request, num, acronym):
     meeting = get_meeting(num=num,type_in=None)
@@ -2367,13 +2364,14 @@ def upload_session_bluesheets(request, session_id, num):
             ota = session.official_timeslotassignment()
             sess_time = ota and ota.timeslot.time
             if not sess_time:
-                return HttpResponse("Cannot receive uploads for an unscheduled session.  Please check the session ID.", status=410, content_type="text/plain")
+                return HttpResponseGone("Cannot receive uploads for an unscheduled session.  Please check the session ID.", content_type="text/plain")
 
 
             save_error = save_bluesheet(request, session, file, encoding=form.file_encoding[file.name])
             if save_error:
                 form.add_error(None, save_error)
             else:
+                messages.success(request, 'Successfully uploaded bluesheets.')
                 return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
     else: 
         form = UploadBlueSheetForm()
@@ -2455,62 +2453,29 @@ def upload_session_minutes(request, session_id, num):
             apply_to_all = session.type_id == 'regular'
             if show_apply_to_all_checkbox:
                 apply_to_all = form.cleaned_data['apply_to_all']
-            if minutes_sp:
+
-                doc = minutes_sp.document
+            # Set up the new revision
-                doc.rev = '%02d' % (int(doc.rev)+1)
+            try:
-                minutes_sp.rev = doc.rev
+                save_session_minutes_revision(
-                minutes_sp.save()
+                    session=session,
+                    apply_to_all=apply_to_all,
+                    file=file,
+                    ext=ext,
+                    encoding=form.file_encoding[file.name],
+                    request=request,
+                )
+            except SessionNotScheduledError:
+                return HttpResponseGone(
+                    "Cannot receive uploads for an unscheduled session. Please check the session ID.",
+                    content_type="text/plain",
+                )
+            except SaveMaterialsError as err:
+                form.add_error(None, str(err))
             else:
-                ota = session.official_timeslotassignment()
+                # no exception -- success!
-                sess_time = ota and ota.timeslot.time
+                messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.')
-                if not sess_time:
+                return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
-                    return HttpResponse("Cannot receive uploads for an unscheduled session.  Please check the session ID.", status=410, content_type="text/plain")
+    else:
-                if session.meeting.type_id=='ietf':
-                    name = 'minutes-%s-%s' % (session.meeting.number, 
-                                                 session.group.acronym) 
-                    title = 'Minutes IETF%s: %s' % (session.meeting.number, 
-                                                         session.group.acronym) 
-                    if not apply_to_all:
-                        name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),)
-                        title += ': %s' % (sess_time.strftime("%a %H:%M"),)
-                else:
-                    name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M"))
-                    title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M"))
-                if Document.objects.filter(name=name).exists():
-                    doc = Document.objects.get(name=name)
-                    doc.rev = '%02d' % (int(doc.rev)+1)
-                else:
-                    doc = Document.objects.create(
-                              name = name,
-                              type_id = 'minutes',
-                              title = title,
-                              group = session.group,
-                              rev = '00',
-                          )
-                    DocAlias.objects.create(name=doc.name).docs.add(doc)
-                doc.states.add(State.objects.get(type_id='minutes',slug='active'))
-                if session.sessionpresentation_set.filter(document=doc).exists():
-                    sp = session.sessionpresentation_set.get(document=doc)
-                    sp.rev = doc.rev
-                    sp.save()
-                else:
-                    session.sessionpresentation_set.create(document=doc,rev=doc.rev)
-            if apply_to_all:
-                for other_session in sessions:
-                    if other_session != session:
-                        other_session.sessionpresentation_set.filter(document__type='minutes').delete()
-                        other_session.sessionpresentation_set.create(document=doc,rev=doc.rev)
-            filename = '%s-%s%s'% ( doc.name, doc.rev, ext)
-            doc.uploaded_filename = filename
-            e = NewRevisionDocEvent.objects.create(doc=doc, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev, rev=doc.rev)
-            # The way this function builds the filename it will never trigger the file delete in handle_file_upload.
-            save_error = handle_upload_file(file, filename, session.meeting, 'minutes', request=request, encoding=form.file_encoding[file.name])
-            if save_error:
-                form.add_error(None, save_error)
-            else:
-                doc.save_with_history([e])
-                return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
-    else: 
         form = UploadMinutesForm(show_apply_to_all_checkbox)
 
     return render(request, "meeting/upload_session_minutes.html", 
@@ -2532,7 +2497,7 @@ def upload_session_agenda(request, session_id, num):
 
     session_number = None
     sessions = get_sessions(session.meeting.number,session.group.acronym)
-    show_apply_to_all_checkbox = len(sessions) > 1 if session.type_id == 'regular' else False
+    show_apply_to_all_checkbox = len(sessions) > 1 if session.type.slug == 'regular' else False
     if len(sessions) > 1:
        session_number = 1 + sessions.index(session)
 
@@ -2543,7 +2508,7 @@ def upload_session_agenda(request, session_id, num):
         if form.is_valid():
             file = request.FILES['file']
             _, ext = os.path.splitext(file.name)
-            apply_to_all = session.type_id == 'regular'
+            apply_to_all = session.type.slug == 'regular'
             if show_apply_to_all_checkbox:
                 apply_to_all = form.cleaned_data['apply_to_all']
             if agenda_sp:
@@ -2555,7 +2520,7 @@ def upload_session_agenda(request, session_id, num):
                 ota = session.official_timeslotassignment()
                 sess_time = ota and ota.timeslot.time
                 if not sess_time:
-                    return HttpResponse("Cannot receive uploads for an unscheduled session.  Please check the session ID.", status=410, content_type="text/plain")
+                    return HttpResponseGone("Cannot receive uploads for an unscheduled session.  Please check the session ID.", content_type="text/plain")
                 if session.meeting.type_id=='ietf':
                     name = 'agenda-%s-%s' % (session.meeting.number, 
                                                  session.group.acronym) 
@@ -2603,6 +2568,7 @@ def upload_session_agenda(request, session_id, num):
                 form.add_error(None, save_error)
             else:
                 doc.save_with_history([e])
+                messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.')
                 return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
     else: 
         form = UploadAgendaForm(show_apply_to_all_checkbox, initial={'apply_to_all':session.type_id=='regular'})
@@ -2698,6 +2664,9 @@ def upload_session_slides(request, session_id, num, name):
             else:
                 doc.save_with_history([e])
                 post_process(doc)
+                messages.success(
+                    request,
+                    f'Successfully uploaded slides as revision {doc.rev} of {doc.name}.')
                 return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
     else: 
         initial = {}
@@ -2764,6 +2733,7 @@ def propose_session_slides(request, session_id, num):
             msg.by = request.user.person
             msg.save()
             send_mail_message(request, msg)
+            messages.success(request, 'Successfully submitted proposed slides.')
             return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
     else: 
         initial = {}
@@ -2787,6 +2757,7 @@ def remove_sessionpresentation(request, session_id, num, name):
         c = DocEvent(type="added_comment", doc=sp.document, rev=sp.document.rev, by=request.user.person)
         c.desc = "Removed from session: %s" % (session)
         c.save()
+        messages.success(request, f'Successfully removed {name}.')
         return redirect('ietf.meeting.views.session_details', num=session.meeting.number, acronym=session.group.acronym)
 
     return render(request,'meeting/remove_sessionpresentation.html', {'sp': sp })
@@ -4119,3 +4090,87 @@ def approve_proposed_slides(request, slidesubmission_id, num):
                    'existing_doc' : existing_doc,
                    'form': form,
                   })
+
+
+def import_session_minutes(request, session_id, num):
+    """Import session minutes from the ietf.notes.org site
+
+    A GET pulls in the markdown for a session's notes using the HedgeDoc API. An HTML preview of how
+    the datatracker will render the result is sent back. The confirmation form presented to the user
+    contains a hidden copy of the markdown source that will be submitted back if approved.
+
+    A POST accepts the hidden source and creates a new revision of the notes. This step does *not*
+    retrieve the note from the HedgeDoc API again - it posts the hidden source from the form. Any
+    changes to the HedgeDoc site after the preview was retrieved will be ignored. We could also pull
+    the source again and re-display the updated preview with an explanatory message, but there will
+    always be a race condition. Rather than add that complication, we assume that the user previewing
+    the imported minutes will be aware of anyone else changing the notes and coordinate with them.
+
+    A consequence is that the user could tamper with the hidden form and it would be accepted. This is
+    ok, though, because they could more simply upload whatever they want through the upload form with
+    the same effect so no exploit is introduced.
+    """
+    session = get_object_or_404(Session, pk=session_id)
+    note = Note(session.notes_id())
+
+    if not session.can_manage_materials(request.user):
+        permission_denied(request, "You don't have permission to import minutes for this session.")
+    if session.is_material_submission_cutoff() and not has_role(request.user, "Secretariat"):
+        permission_denied(request, "The materials cutoff for this session has passed. Contact the secretariat for further action.")
+
+    if request.method == 'POST':
+        form = ImportMinutesForm(request.POST)
+        if not form.is_valid():
+            import_contents = form.data['markdown_text']
+        else:
+            import_contents = form.cleaned_data['markdown_text']
+            try:
+                save_session_minutes_revision(
+                    session=session,
+                    file=io.BytesIO(import_contents.encode('utf8')),
+                    ext='.md',
+                    request=request,
+                )
+            except SessionNotScheduledError:
+                return HttpResponseGone(
+                    "Cannot import minutes for an unscheduled session. Please check the session ID.",
+                    content_type="text/plain",
+                )
+            except SaveMaterialsError as err:
+                form.add_error(None, str(err))
+            else:
+                messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.')
+                return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
+    else:
+        try:
+            import_contents = note.get_source()
+        except NoteError as err:
+            messages.error(request, f'Could not import notes with id {note.id}: {err}.')
+            return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
+        form = ImportMinutesForm(initial={'markdown_text': import_contents})
+
+    # Try to prevent pointless revision creation. Note that we do not block replacing
+    # a document with an identical copy in the validation above. We cannot entirely
+    # avoid a race condition and the likelihood/amount of damage is very low so no
+    # need to complicate things further.
+    current_minutes = session.minutes()
+    contents_changed = True
+    if current_minutes:
+        try:
+            with open(current_minutes.get_file_name()) as f:
+                if import_contents == Note.preprocess_source(f.read()):
+                    contents_changed = False
+                    messages.warning(request, 'This document is identical to the current revision, no need to import.')
+        except FileNotFoundError:
+            pass  # allow import if the file is missing
+
+    return render(
+        request,
+        'meeting/import_minutes.html',
+        {
+            'form': form,
+            'note': note,
+            'session': session,
+            'contents_changed': contents_changed,
+        },
+    )

ietf/meeting/views_proceedings.py

@@ -14,7 +14,7 @@ from ietf.meeting.forms import FileUploadForm
 from ietf.meeting.models import Meeting, MeetingHost
 from ietf.meeting.helpers import get_meeting
 from ietf.name.models import ProceedingsMaterialTypeName
-from ietf.secr.proceedings.utils import handle_upload_file
+from ietf.meeting.utils import handle_upload_file
 from ietf.utils.text import xslugify
 
 class UploadProceedingsMaterialForm(FileUploadForm):

ietf/secr/meetings/views.py

@@ -20,14 +20,13 @@ from ietf.utils.mail import send_mail
 from ietf.meeting.forms import duration_string
 from ietf.meeting.helpers import get_meeting, make_materials_directories, populate_important_dates
 from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent
-from ietf.meeting.utils import add_event_info_to_session_qs
+from ietf.meeting.utils import add_event_info_to_session_qs, handle_upload_file
 from ietf.name.models import SessionStatusName
 from ietf.group.models import Group, GroupEvent
 from ietf.secr.meetings.blue_sheets import create_blue_sheets
 from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm,
     MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm,
     UploadBlueSheetForm, MeetingRoomOptionsForm )
-from ietf.secr.proceedings.utils import handle_upload_file
 from ietf.secr.sreq.views import get_initial_session
 from ietf.secr.utils.meeting import get_session, get_timeslot
 from ietf.mailtrigger.utils import gather_address_lists

ietf/secr/proceedings/utils.py

@@ -1,74 +0,0 @@
-# Copyright The IETF Trust 2016-2019, All Rights Reserved
-
-import glob
-import io
-import os
-
-from django.conf import settings
-from django.contrib import messages
-from django.utils.encoding import smart_text
-
-import debug                            # pyflakes:ignore
-
-from ietf.utils.html import sanitize_document
-
-def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None):
-    '''
-    This function takes a file object, a filename and a meeting object and subdir as string.
-    It saves the file to the appropriate directory, get_materials_path() + subdir.
-    If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
-    zip file and unzips the file in the new directory.
-    '''
-    base, extension = os.path.splitext(filename)
-
-    if extension == '.zip':
-        path = os.path.join(meeting.get_materials_path(),subdir,base)
-        if not os.path.exists(path):
-            os.mkdir(path)
-    else:
-        path = os.path.join(meeting.get_materials_path(),subdir)
-        if not os.path.exists(path):
-            os.makedirs(path)
-
-    # agendas and minutes can only have one file instance so delete file if it already exists
-    if subdir in ('agenda','minutes'):
-        old_files = glob.glob(os.path.join(path,base) + '.*')
-        for f in old_files:
-            os.remove(f)
-
-    destination = io.open(os.path.join(path,filename), 'wb+')
-    if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
-        file.open()
-        text = file.read()
-        if encoding:
-            try:
-                text = text.decode(encoding)
-            except LookupError as e:
-                return "Failure trying to save '%s': Could not identify the file encoding, got '%s'.  Hint: Try to upload as UTF-8." % (filename, str(e)[:120])
-        else:
-            try:
-                text = smart_text(text)
-            except UnicodeDecodeError as e:
-                return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
-        # Whole file sanitization; add back what's missing from a complete
-        # document (sanitize will remove these).
-        clean = sanitize_document(text)
-        destination.write(clean.encode('utf8'))
-        if request and clean != text:
-            messages.warning(request, "Uploaded html content is sanitized to prevent unsafe content.  "
-                                      "Your upload %s was changed by the sanitization; please check the "
-                                       "resulting content.  " % (filename, ))
-    else:
-        if hasattr(file, 'chunks'):
-            for chunk in file.chunks():
-                destination.write(chunk)
-        else:
-            destination.write(file.read())
-    destination.close()
-
-    # unzip zipfile
-    if extension == '.zip':
-        os.chdir(path)
-        os.system('unzip %s' % filename)
-
-    return None

ietf/settings.py

@@ -148,6 +148,7 @@ IETF_ID_URL = IETF_HOST_URL + 'id/' # currently unused
 IETF_ID_ARCHIVE_URL = IETF_HOST_URL + 'archive/id/'
 IETF_AUDIO_URL = IETF_HOST_URL + 'audio/'
 
+IETF_NOTES_URL = 'https://notes.ietf.org/'  # HedgeDoc base URL
 
 # Absolute path to the directory static files should be collected to.
 # Example: "/var/www/example.com/static/"
@@ -996,6 +997,7 @@ DOT_BINARY = '/usr/bin/dot'
 UNFLATTEN_BINARY= '/usr/bin/unflatten'
 RSYNC_BINARY = '/usr/bin/rsync'
 YANGLINT_BINARY = '/usr/bin/yanglint'
+DE_GFM_BINARY = '/usr/bin/de-gfm.ruby2.5'
 
 # Account settings
 DAYS_TO_EXPIRE_REGISTRATION_LINK = 3

ietf/templates/meeting/import_minutes.html

@@ -0,0 +1,36 @@
+{% extends "base.html" %}
+{% load bootstrap3 origin %}
+
+{% block morecss %}
+  #preview { width: 100%; height: 60vh; border: solid 2px; }
+{% endblock %}
+
+{% block title %}
+  Import Preview: {% firstof note.get_title note.id %}
+{% endblock %}
+
+{% block content %}
+  {% origin %}
+  <h1>Import Preview: {% firstof note.get_title note.id %}</h1>
+  <div class="row">
+    <div class="col-md-11">
+      <p class="pull-left">Last updated: {% firstof note.get_update_time "unknown" %}</p>
+      <p class="pull-right"><a href="{{ note.url }}">View on notes.ietf.org</a></p>
+      <iframe id="preview" srcdoc="{{ note.get_preview|force_escape }}" sandbox>
+        <p>Your browser does not support iframes, so preview cannot be shown.</p>
+      </iframe>
+    </div>
+  </div>
+  <div class="row">
+    <div class="col-md-11">
+      <form method="post" enctype="multipart/form-data">
+        {% csrf_token %}
+        {% bootstrap_form form %}
+        {% buttons %}
+          <button type="submit" class="btn btn-primary pull-left" {% if not contents_changed %}disabled{% endif %}> Import </button>
+          <a class="btn btn-default pull-right" href="{% url 'ietf.meeting.views.session_details' num=session.meeting.number acronym=session.group.acronym %}"> Back </a>
+        {% endbuttons %}
+      </form>
+    </div>
+  </div>
+{% endblock %}

ietf/templates/meeting/interim_session_buttons.html

@@ -16,11 +16,7 @@
     {% endif %}
       <!-- etherpad -->
     {% if use_codimd %}
-      {% if item.slot_type.slug == 'plenary' %}
+      <a class="" href="{{ session.notes_url }}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-        <a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-plenary" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-      {% else %}
-        <a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-{{acronym}}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-      {% endif %}
     {% endif %}
 
   {# show stream buttons up till end of session, then show archive buttons #}

ietf/templates/meeting/session_buttons_include.html

@@ -22,11 +22,7 @@
 
   <!-- HedgeDoc -->
   {% if use_codimd %}
-    {% if timeslot.type.slug == 'plenary' %}
+    <a class="" href="{{ session.notes_url }}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-      <a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-plenary" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-    {% else %}
-      <a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-{{acronym}}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
-    {% endif %}
   {% endif %}
 
   {# show stream buttons up till end of session, then show archive buttons #}

ietf/templates/meeting/session_details_panel.html

@@ -74,6 +74,9 @@
                     {% endif %}
                     {% if pres.document.type.slug != 'bluesheets' or user|has_role:"Secretariat" or meeting.type.slug == 'interim' and can_manage_materials %}
                       <a class="btn btn-default btn-sm pull-right" href="{{upload_url}}">Upload Revision</a>
+                      {% if pres.document.type.slug == 'minutes' %}
+                        <a class="btn btn-default btn-sm pull-right" href="{% url 'ietf.meeting.views.import_session_minutes' num=session.meeting.number session_id=session.pk %}">Import from notes.ietf.org</a>
+                      {% endif %}
                     {% endif %}
                   </td> 
                 {% endif %}

ietf/utils/hedgedoc.py

@@ -0,0 +1,117 @@
+# Copyright The IETF Trust 2021, All Rights Reserved
+# -*- coding: utf-8 -*-
+"""HedgeDoc API utilities"""
+import json
+import requests
+import subprocess
+import debug  # pyflakes: ignore
+
+from urllib.parse import urljoin
+from django.conf import settings
+
+from ietf.utils.markdown import markdown
+
+
+class Note:
+    base_url = settings.IETF_NOTES_URL
+
+    def __init__(self, id):
+        self.id = id
+        self.url = urljoin(self.base_url, self.id)  # URL on notes site
+        self._metadata = None
+        self._preview = None
+        self._source = None
+
+    @classmethod
+    def preprocess_source(cls, raw_source):
+        """Perform preprocessing on raw source input
+
+        Guaranteed to process input in the same way that the Note class processes
+        markdown source pulled from the notes site.
+        """
+        return de_gfm(raw_source)
+
+    def get_source(self):
+        """Retrieve markdown source from hedgedoc
+
+        Converts line breaks from GitHub Flavored Markdown (GFM) style to
+        to traditional markdown.
+        """
+        if self._source is None:
+            try:
+                r = requests.get(urljoin(self.base_url, f'{self.id}/download'), allow_redirects=True)
+            except requests.RequestException:
+                raise ServerNoteError
+            if r.status_code != 200:
+                raise NoteNotFound
+            self._source = self.preprocess_source(r.text)
+        return self._source
+
+    def get_preview(self):
+        if self._preview is None:
+            self._preview = markdown(self.get_source())
+        return self._preview
+
+    def get_title(self):
+        try:
+            metadata = self._retrieve_metadata()
+        except NoteError:
+            metadata = {}  # don't let an error retrieving the title prevent retrieval
+        return metadata.get('title', None)
+
+    def get_update_time(self):
+        try:
+            metadata = self._retrieve_metadata()
+        except NoteError:
+            metadata = {}  # don't let an error retrieving the update timestamp prevent retrieval
+        return metadata.get('updatetime', None)
+
+    def _retrieve_metadata(self):
+        if self._metadata is None:
+            try:
+                r = requests.get(urljoin(self.base_url, f'{self.id}/info'), allow_redirects=True)
+            except requests.RequestException:
+                raise ServerNoteError
+            if r.status_code != 200:
+                raise NoteNotFound
+            try:
+                self._metadata = json.loads(r.content)
+            except json.JSONDecodeError:
+                raise InvalidNote
+        return self._metadata
+
+
+def de_gfm(source: str):
+    """Convert GFM line breaks to standard Markdown
+
+    Calls de-gfm from the kramdown-rfc2629 gem.
+    """
+    result = subprocess.run(
+        [settings.DE_GFM_BINARY,],
+        stdout=subprocess.PIPE,  # post-Python 3.7, this can be replaced with capture_output=True
+        input=source,
+        encoding='utf8',
+        check=True,
+    )
+    return result.stdout
+
+
+class NoteError(Exception):
+    """Base class for exceptions in this module"""
+    default_message = 'A note-related error occurred'
+
+    def __init__(self, *args, **kwargs):
+        if not args:
+            args = (self.default_message, )
+        super().__init__(*args)
+
+
+class ServerNoteError(NoteError):
+    default_message = 'Could not reach the notes server'
+
+class NoteNotFound(NoteError):
+    default_message = 'Note did not exist or could not be loaded'
+
+
+class InvalidNote(NoteError):
+    default_message = 'Note data invalid'

ietf/utils/markdown.py

@@ -0,0 +1,20 @@
+# Copyright The IETF Trust 2021, All Rights Reserved
+# -*- coding: utf-8 -*-
+"""Markdown wrapper
+
+Use this instead of importing markdown directly to guarantee consistent extensions / options through
+the datatracker.
+"""
+import bleach
+import markdown as python_markdown
+
+from django.utils.safestring import mark_safe
+from markdown.extensions.extra import ExtraExtension
+
+ALLOWED_TAGS = bleach.ALLOWED_TAGS + ['p', 'h1', 'h2', 'h3', 'h4', 'br']
+
+def markdown(text):
+    return mark_safe(bleach.clean(
+        python_markdown.markdown(text, extensions=[ExtraExtension()]),
+        tags=ALLOWED_TAGS,
+    ))

ietf/utils/test_utils.py

@@ -38,6 +38,7 @@ import os
 import re
 import email
 import html5lib
+import requests_mock
 import shutil
 import sys
 
@@ -151,6 +152,7 @@ class ReverseLazyTest(django.test.TestCase):
         response = self.client.get('/ipr/update/')
         self.assertRedirects(response, "/ipr/", status_code=301)
 
+
 class TestCase(django.test.TestCase):
     """IETF TestCase class
 
@@ -158,6 +160,7 @@ class TestCase(django.test.TestCase):
       * asserts for html5 validation.
       * tempdir() convenience method
       * setUp() and tearDown() that override settings paths with temp directories
+      * mocking the requests library to prevent dependencies on the outside network
 
     The setUp() and tearDown() methods create / remove temporary paths and override
     Django's settings with the temp dir names. Subclasses of this class must
@@ -165,6 +168,12 @@ class TestCase(django.test.TestCase):
     anew for each test to avoid risk of cross-talk between test cases. Overriding
     the settings_temp_path_overrides class value will modify which path settings are
     replaced with temp test dirs.
+
+    Uses requests-mock to prevent the requests library from making requests to outside
+    resources. The requests-mock library allows nested mocks, so individual tests can
+    ignore this. Note that the mock set up by this class will intercept any requests
+    not handled by a test's inner mock - even if the latter is created with
+    real_http=True.
     """
     # These settings will be overridden with empty temporary directories
     settings_temp_path_overrides = [
@@ -257,10 +266,14 @@ class TestCase(django.test.TestCase):
     def __str__(self):
         return u"%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName)
 
-
     def setUp(self):
-        # Replace settings paths with temporary directories.
         super().setUp()
+
+        # Prevent the requests library from making live requests during tests
+        self.requests_mock = requests_mock.Mocker()
+        self.requests_mock.start()
+
+        # Replace settings paths with temporary directories.
         self._ietf_temp_dirs = {}  # trashed during tearDown, DO NOT put paths you care about in this
         for setting in self.settings_temp_path_overrides:
             self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting))
@@ -271,4 +284,5 @@ class TestCase(django.test.TestCase):
         self._ietf_saved_context.disable()
         for dir in self._ietf_temp_dirs.values():
             shutil.rmtree(dir)
+        self.requests_mock.stop()
         super().tearDown()

ietf/utils/tests_hedgedoc.py

@@ -0,0 +1,45 @@
+# Copyright The IETF Trust 2021, All Rights Reserved
+# -*- coding: utf-8 -*-
+"""HedgeDoc API utilities tests"""
+import requests_mock
+
+from ietf.utils.tests import TestCase
+from ietf.utils.hedgedoc import Note
+
+
+class NoteTests(TestCase):
+    SAMPLE_MARKDOWN = ''.join((
+        '# Standard Markdown\n',
+        'This is a small sample of markdown text. It uses GFM-style line breaks.\n',
+        '\n',
+        'This is a second paragraph.\n',
+        'It has line breaks in GFM style.\n',
+        'And also some standard line breaks.  \n',
+        'That is all.\n',
+    ))
+    SAMPLE_MARKDOWN_OUTPUT = ''.join((
+        '# Standard Markdown   {#standard-markdown}\n',
+        '\n',
+        'This is a small sample of markdown text. It uses GFM-style line breaks.\n',
+        '\n',
+        'This is a second paragraph.  \n',
+        'It has line breaks in GFM style.  \n',
+        'And also some standard line breaks.   \n',
+        'That is all.\n',
+        '\n',
+    ))
+
+    def test_retrieves_note(self):
+        with requests_mock.Mocker() as mock:
+            mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN)
+            n = Note('my_id')
+            result = n.get_source()
+        self.assertEqual(result, self.SAMPLE_MARKDOWN_OUTPUT)
+
+    def test_uses_preprocess_class_method(self):
+        """Imported text should be processed by the preprocess_source class method"""
+        with requests_mock.Mocker() as mock:
+            mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN)
+            n = Note('my_id')
+            result = n.get_source()
+        self.assertEqual(result, Note.preprocess_source(self.SAMPLE_MARKDOWN))