altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

Add ability to import session minutes from notes.ietf.org. Mock out calls to the requests library in tests. Call markdown library through a util method. Fixes #3489. Commit ready for merge.

Browse source 
- Legacy-Id: 19763
This commit is contained in:
Jennifer Richards 2021-12-09 17:16:19 +00:00
parent b04254a293
commit fd0df6f619
21 changed files with 750 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ietf/doc/views_bofreq.py
View file

@ -3,7 +3,6 @@
import debug # pyflakes:ignore
import io
import markdown
from django import forms
from django.contrib.auth.decorators import login_required
@ -20,6 +19,7 @@ from ietf.doc.utils import add_state_change_event
from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible
from ietf.ietfauth.utils import has_role, role_required
from ietf.person.fields import SearchablePersonsField
from ietf.utils import markdown
from ietf.utils.response import permission_denied
from ietf.utils.text import xslugify
from ietf.utils.textupload import get_cleaned_text_file_content
@ -64,7 +64,7 @@ class BofreqUploadForm(forms.Form):
if require_field("bofreq_file"):
content = get_cleaned_text_file_content(self.cleaned_data["bofreq_file"])
try:
_ = markdown.markdown(content, extensions=['extra'])
_ = markdown.markdown(content)
except Exception as e:
raise forms.ValidationError(f'Markdown processing failed: {e}')

7
ietf/doc/views_doc.py
View file

@ -40,7 +40,6 @@ import io
import json
import os
import re
import markdown
from urllib.parse import quote
@ -80,7 +79,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
from ietf.review.models import ReviewAssignment
from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
from ietf.review.utils import no_review_from_teams_on_doc
from ietf.utils import markup_txt, log
from ietf.utils import markup_txt, log, markdown
from ietf.utils.draft import Draft
from ietf.utils.response import permission_denied
from ietf.utils.text import maybe_split
@ -550,7 +549,7 @@ def document_main(request, name, rev=None):
))
if doc.type_id == "bofreq":
content = markdown.markdown(doc.text_or_error(),extensions=['extra'])
content = markdown.markdown(doc.text_or_error())
editors = bofreq_editors(doc)
responsible = bofreq_responsible(doc)
can_manage = has_role(request.user,['Secretariat', 'Area Director', 'IAB'])
@ -661,7 +660,7 @@ def document_main(request, name, rev=None):
content = doc.text_or_error()
t = "plain text"
elif extension == ".md":
content = markdown.markdown(doc.text_or_error(), extensions=['extra'])
content = markdown.markdown(doc.text_or_error())
content_is_html = True
t = "markdown"
other_types.append((t, url))

5
ietf/group/views.py
View file

@ -38,7 +38,6 @@ import copy
import datetime
import itertools
import io
import markdown
import math
import os
import re
@ -121,7 +120,7 @@ from ietf.settings import MAILING_LIST_INFO_URL
from ietf.utils.pipe import pipe
from ietf.utils.response import permission_denied
from ietf.utils.text import strip_suffix
from ietf.utils import markdown
# --- Helpers ----------------------------------------------------------
@ -581,7 +580,7 @@ def group_about_rendertest(request, acronym, group_type=None):
if group.charter:
charter = get_charter_text(group)
try:
rendered = markdown.markdown(charter, extensions=['extra'])
rendered = markdown.markdown(charter)
except Exception as e:
rendered = f'Markdown rendering failed: {e}'
return render(request, 'group/group_about_rendertest.html', {'group':group, 'charter':charter, 'rendered':rendered})

4
ietf/meeting/forms.py
View file

@ -436,6 +436,10 @@ class UploadSlidesForm(ApplyToAllFileUploadForm):
return title
class ImportMinutesForm(forms.Form):
markdown_text = forms.CharField(strip=False, widget=forms.HiddenInput)
class RequestMinutesForm(forms.Form):
to = MultiEmailField()
cc = MultiEmailField(required=False)

8
ietf/meeting/models.py
View file

@ -14,6 +14,7 @@ import string
from collections import namedtuple
from pathlib import Path
from urllib.parse import urljoin
import debug # pyflakes:ignore
@ -1260,6 +1261,13 @@ class Session(models.Model):
else:
return self.group.acronym
def notes_id(self):
note_id_fragment = 'plenary' if self.type.slug == 'plenary' else self.group.acronym
return f'notes-ietf-{self.meeting.number}-{note_id_fragment}'
def notes_url(self):
return urljoin(settings.IETF_NOTES_URL, self.notes_id())
class SchedulingEvent(models.Model):
session = ForeignKey(Session)
time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened")

187
ietf/meeting/tests_views.py
View file

@ -8,6 +8,8 @@ import random
import re
import shutil
import pytz
import requests.exceptions
import requests_mock
from unittest import skipIf
from mock import patch, PropertyMock
@ -19,7 +21,6 @@ from urllib.parse import urlparse, urlsplit
from PIL import Image
from pathlib import Path
from django.urls import reverse as urlreverse
from django.conf import settings
from django.contrib.auth.models import User
@ -5405,12 +5406,16 @@ class IphoneAppJsonTests(TestCase):
self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
class FinalizeProceedingsTests(TestCase):
@patch('urllib.request.urlopen')
def test_finalize_proceedings(self, mock_urlopen):
mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
@override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
@requests_mock.Mocker()
def test_finalize_proceedings(self, mock):
make_meeting_test_data()
meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last()
meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None)
mock.get(
settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
)
url = urlreverse('ietf.meeting.views.finalize_proceedings',kwargs={'num':meeting.number})
login_testing_unauthorized(self,"secretary",url)
@ -5605,8 +5610,10 @@ class MaterialsTests(TestCase):
self.assertEqual(doc.rev,'02')
# Verify that we don't have dead links
url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
top = '/meeting/%s/' % session.meeting.number
self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
self.crawl_materials(url=url, top=top)
def test_upload_minutes_agenda_unscheduled(self):
@ -5653,8 +5660,10 @@ class MaterialsTests(TestCase):
self.assertEqual(doc.rev,'00')
# Verify that we don't have dead links
url = url=urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
url = urlreverse('ietf.meeting.views.session_details', kwargs={'num':session.meeting.number, 'acronym': session.group.acronym})
top = '/meeting/%s/' % session.meeting.number
self.requests_mock.get(f'{session.notes_url()}/download', text='markdown notes')
self.requests_mock.get(f'{session.notes_url()}/info', text=json.dumps({'title': 'title', 'updatetime': '2021-12-01T17:11:00z'}))
self.crawl_materials(url=url, top=top)
def test_upload_slides(self):
@ -5928,6 +5937,151 @@ class MaterialsTests(TestCase):
self.assertIn('third version', contents)
@override_settings(IETF_NOTES_URL='https://notes.ietf.org/')
class ImportNotesTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
def setUp(self):
super().setUp()
self.session = SessionFactory(meeting__type_id='ietf')
self.meeting = self.session.meeting
def test_retrieves_note(self):
"""Can import and preview a note from notes.ietf.org"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
with requests_mock.Mocker() as mock:
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
iframe = q('iframe#preview')
self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
markdown_text_input = q('form #id_markdown_text')
self.assertEqual(markdown_text_input.val(), 'markdown text')
def test_retrieves_with_broken_metadata(self):
"""Can import and preview a note even if it has a metadata problem"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
with requests_mock.Mocker() as mock:
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='markdown text')
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info', text='this is not valid json {]')
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
iframe = q('iframe#preview')
self.assertEqual('<p>markdown text</p>', iframe.attr('srcdoc'))
markdown_text_input = q('form #id_markdown_text')
self.assertEqual(markdown_text_input.val(), 'markdown text')
def test_redirects_on_success(self):
"""Redirects to session details page after import"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
r = self.client.post(url, {'markdown_text': 'markdown text'})
self.assertRedirects(
r,
urlreverse(
'ietf.meeting.views.session_details',
kwargs={
'num': self.meeting.number,
'acronym': self.session.group.acronym,
},
),
)
def test_imports_previewed_text(self):
"""Import text that was shown as preview even if notes site is updated"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
with requests_mock.Mocker() as mock:
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='updated markdown text')
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
r = self.client.post(url, {'markdown_text': 'original markdown text'})
self.assertEqual(r.status_code, 302)
minutes_path = Path(self.meeting.get_materials_path()) / 'minutes'
with (minutes_path / self.session.minutes().uploaded_filename).open() as f:
self.assertEqual(f.read(), 'original markdown text')
def test_refuses_identical_import(self):
"""Should not be able to import text identical to the current revision"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
r = self.client.post(url, {'markdown_text': 'original markdown text'}) # create a rev
self.assertEqual(r.status_code, 302)
with requests_mock.Mocker() as mock:
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
r = self.client.get(url) # try to import the same text
self.assertContains(r, "This document is identical", status_code=200)
q = PyQuery(r.content)
self.assertEqual(len(q('button:disabled[type="submit"]')), 1)
self.assertEqual(len(q('button:not(:disabled)[type="submit"]')), 0)
def test_handles_missing_previous_revision_file(self):
"""Should still allow import if the file for the previous revision is missing"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
r = self.client.post(url, {'markdown_text': 'original markdown text'}) # create a rev
# remove the file uploaded for the first rev
minutes_docs = self.session.sessionpresentation_set.filter(document__type='minutes')
self.assertEqual(minutes_docs.count(), 1)
Path(minutes_docs.first().document.get_file_name()).unlink()
self.assertEqual(r.status_code, 302)
with requests_mock.Mocker() as mock:
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/download', text='original markdown text')
mock.get(f'https://notes.ietf.org/{self.session.notes_id()}/info',
text=json.dumps({"title": "title", "updatetime": "2021-12-02T11:22:33z"}))
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
iframe = q('iframe#preview')
self.assertEqual('<p>original markdown text</p>', iframe.attr('srcdoc'))
markdown_text_input = q('form #id_markdown_text')
self.assertEqual(markdown_text_input.val(), 'original markdown text')
def test_handles_note_does_not_exist(self):
"""Should not try to import a note that does not exist"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
with requests_mock.Mocker() as mock:
mock.get(requests_mock.ANY, status_code=404)
r = self.client.get(url, follow=True)
self.assertContains(r, 'Could not import', status_code=200)
def test_handles_notes_server_failure(self):
"""Problems communicating with the notes server should be handled gracefully"""
url = urlreverse('ietf.meeting.views.import_session_minutes',
kwargs={'num': self.meeting.number, 'session_id': self.session.pk})
self.client.login(username='secretary', password='secretary+password')
with requests_mock.Mocker() as mock:
mock.get(re.compile(r'.+/download'), exc=requests.exceptions.ConnectTimeout)
mock.get(re.compile(r'.+//info'), text='{}')
r = self.client.get(url, follow=True)
self.assertContains(r, 'Could not reach the notes server', status_code=200)
class SessionTests(TestCase):
def test_meeting_requests(self):
@ -6911,12 +7065,15 @@ class ProceedingsTests(BaseMeetingTestCase):
0,
)
@patch('ietf.meeting.utils.requests.get')
def test_proceedings_attendees(self, mockobj):
mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
@override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
@requests_mock.Mocker()
def test_proceedings_attendees(self, mock):
make_meeting_test_data()
meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
mock.get(
settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
)
finalize(meeting)
url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':97})
response = self.client.get(url)
@ -6924,14 +7081,18 @@ class ProceedingsTests(BaseMeetingTestCase):
q = PyQuery(response.content)
self.assertEqual(1,len(q("#id_attendees tbody tr")))
@patch('urllib.request.urlopen')
def test_proceedings_overview(self, mock_urlopen):
@override_settings(STATS_REGISTRATION_ATTENDEES_JSON_URL='https://ietf.example.com/{number}')
@requests_mock.Mocker()
def test_proceedings_overview(self, mock):
'''Test proceedings IETF Overview page.
Note: old meetings aren't supported so need to add a new meeting then test.
'''
mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
make_meeting_test_data()
meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="97")
mock.get(
settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number),
text=json.dumps([{"LastName": "Smith", "FirstName": "John", "Company": "ABC", "Country": "US"}]),
)
finalize(meeting)
url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':97})
response = self.client.get(url)

1
ietf/meeting/urls.py
View file

@ -13,6 +13,7 @@ safe_for_all_meeting_types = [
url(r'^session/(?P<session_id>\d+)/bluesheets$', views.upload_session_bluesheets),
url(r'^session/(?P<session_id>\d+)/minutes$', views.upload_session_minutes),
url(r'^session/(?P<session_id>\d+)/agenda$', views.upload_session_agenda),
url(r'^session/(?P<session_id>\d+)/import/minutes$', views.import_session_minutes),
url(r'^session/(?P<session_id>\d+)/propose_slides$', views.propose_session_slides),
url(r'^session/(?P<session_id>\d+)/slides(?:/%(name)s)?$' % settings.URL_REGEXPS, views.upload_session_slides),
url(r'^session/(?P<session_id>\d+)/add_to_session$', views.ajax_add_slides_to_session),

185
ietf/meeting/utils.py
View file

@ -1,17 +1,19 @@
# Copyright The IETF Trust 2016-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import datetime
import itertools
import re
import requests
import subprocess
from collections import defaultdict
from pathlib import Path
from urllib.error import HTTPError
from django.conf import settings
from django.contrib import messages
from django.template.loader import render_to_string
from django.utils.encoding import smart_text
from django.utils.html import format_html
from django.utils.safestring import mark_safe
@ -19,11 +21,14 @@ import debug # pyflakes:ignore
from ietf.dbtemplate.models import DBTemplate
from ietf.meeting.models import Session, SchedulingEvent, TimeSlot, Constraint, SchedTimeSessAssignment
from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
from ietf.group.models import Group
from ietf.group.utils import can_manage_materials
from ietf.name.models import SessionStatusName, ConstraintName
from ietf.person.models import Person
from ietf.secr.proceedings.proc_utils import import_audio_files
from ietf.utils.html import sanitize_document
def session_time_for_sorting(session, use_meeting_date):
official_timeslot = TimeSlot.objects.filter(sessionassignments__session=session, sessionassignments__schedule__in=[session.meeting.schedule, session.meeting.schedule.base if session.meeting.schedule else None]).first()
@ -544,3 +549,179 @@ def preprocess_meeting_important_dates(meetings):
for d in m.important_dates:
d.midnight_cutoff = "UTC 23:59" in d.name.name
def get_meeting_sessions(num, acronym):
types = ['regular','plenary','other']
sessions = Session.objects.filter(
meeting__number=num,
group__acronym=acronym,
type__in=types,
)
if not sessions:
sessions = Session.objects.filter(
meeting__number=num,
short=acronym,
type__in=types,
)
return sessions
class SessionNotScheduledError(Exception):
"""Indicates failure because operation requires a scheduled session"""
pass
class SaveMaterialsError(Exception):
"""Indicates failure saving session materials"""
pass
def save_session_minutes_revision(session, file, ext, request, encoding=None, apply_to_all=False):
"""Creates or updates session minutes records
This updates the database models to reflect a new version. It does not handle
storing the new file contents, that should be handled via handle_upload_file()
or similar.
If the session does not already have minutes, it must be a scheduled
session. If not, SessionNotScheduledError will be raised.
Returns (Document, [DocEvents]), which should be passed to doc.save_with_history()
if the file contents are stored successfully.
"""
minutes_sp = session.sessionpresentation_set.filter(document__type='minutes').first()
if minutes_sp:
doc = minutes_sp.document
doc.rev = '%02d' % (int(doc.rev)+1)
minutes_sp.rev = doc.rev
minutes_sp.save()
else:
ota = session.official_timeslotassignment()
sess_time = ota and ota.timeslot.time
if not sess_time:
raise SessionNotScheduledError
if session.meeting.type_id=='ietf':
name = 'minutes-%s-%s' % (session.meeting.number,
session.group.acronym)
title = 'Minutes IETF%s: %s' % (session.meeting.number,
session.group.acronym)
if not apply_to_all:
name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),)
title += ': %s' % (sess_time.strftime("%a %H:%M"),)
else:
name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M"))
title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M"))
if Document.objects.filter(name=name).exists():
doc = Document.objects.get(name=name)
doc.rev = '%02d' % (int(doc.rev)+1)
else:
doc = Document.objects.create(
name = name,
type_id = 'minutes',
title = title,
group = session.group,
rev = '00',
)
DocAlias.objects.create(name=doc.name).docs.add(doc)
doc.states.add(State.objects.get(type_id='minutes',slug='active'))
if session.sessionpresentation_set.filter(document=doc).exists():
sp = session.sessionpresentation_set.get(document=doc)
sp.rev = doc.rev
sp.save()
else:
session.sessionpresentation_set.create(document=doc,rev=doc.rev)
if apply_to_all:
for other_session in get_meeting_sessions(session.meeting.number, session.group.acronym):
if other_session != session:
other_session.sessionpresentation_set.filter(document__type='minutes').delete()
other_session.sessionpresentation_set.create(document=doc,rev=doc.rev)
filename = f'{doc.name}-{doc.rev}{ext}'
doc.uploaded_filename = filename
e = NewRevisionDocEvent.objects.create(
doc=doc,
by=request.user.person,
type='new_revision',
desc=f'New revision available: {doc.rev}',
rev=doc.rev,
)
# The way this function builds the filename it will never trigger the file delete in handle_file_upload.
save_error = handle_upload_file(
file=file,
filename=doc.uploaded_filename,
meeting=session.meeting,
subdir='minutes',
request=request,
encoding=encoding,
)
if save_error:
raise SaveMaterialsError(save_error)
else:
doc.save_with_history([e])
def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=None):
"""Accept an uploaded materials file
This function takes a file object, a filename and a meeting object and subdir as string.
It saves the file to the appropriate directory, get_materials_path() + subdir.
If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
zip file and unzips the file in the new directory.
"""
filename = Path(filename)
is_zipfile = filename.suffix == '.zip'
path = Path(meeting.get_materials_path()) / subdir
if is_zipfile:
path = path / filename.stem
path.mkdir(parents=True, exist_ok=True)
# agendas and minutes can only have one file instance so delete file if it already exists
if subdir in ('agenda', 'minutes'):
for f in path.glob(f'{filename.stem}.*'):
try:
f.unlink()
except FileNotFoundError:
pass # if the file is already gone, so be it
with (path / filename).open('wb+') as destination:
if filename.suffix in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
file.open()
text = file.read()
if encoding:
try:
text = text.decode(encoding)
except LookupError as e:
return (
f"Failure trying to save '{filename}': "
f"Could not identify the file encoding, got '{str(e)[:120]}'. "
f"Hint: Try to upload as UTF-8."
)
else:
try:
text = smart_text(text)
except UnicodeDecodeError as e:
return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
# Whole file sanitization; add back what's missing from a complete
# document (sanitize will remove these).
clean = sanitize_document(text)
destination.write(clean.encode('utf8'))
if request and clean != text:
messages.warning(request,
(
f"Uploaded html content is sanitized to prevent unsafe content. "
f"Your upload {filename} was changed by the sanitization; "
f"please check the resulting content. "
))
else:
if hasattr(file, 'chunks'):
for chunk in file.chunks():
destination.write(chunk)
else:
destination.write(file.read())
# unzip zipfile
if is_zipfile:
subprocess.call(['unzip', filename], cwd=path)
return None

205
ietf/meeting/views.py
View file

@ -14,7 +14,6 @@ import pytz
import re
import tarfile
import tempfile
import markdown
from calendar import timegm
from collections import OrderedDict, Counter, deque, defaultdict
@ -26,7 +25,7 @@ from django import forms
from django.shortcuts import render, redirect, get_object_or_404
from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden,
HttpResponseNotFound, Http404, HttpResponseBadRequest,
JsonResponse)
JsonResponse, HttpResponseGone)
from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import login_required
@ -57,7 +56,7 @@ from ietf.ietfauth.utils import role_required, has_role, user_is_person
from ietf.mailtrigger.utils import gather_address_lists
from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission
from ietf.meeting.models import SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName
from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm,
from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm,
TimeSlotCreateForm, TimeSlotEditForm, SessionEditForm )
from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name
from ietf.meeting.helpers import get_meeting, get_ietf_meeting, get_current_ietf_meeting_num
@ -76,19 +75,20 @@ from ietf.meeting.helpers import send_interim_announcement_request
from ietf.meeting.utils import finalize, sort_accept_tuple, condition_slide_order
from ietf.meeting.utils import add_event_info_to_session_qs
from ietf.meeting.utils import session_time_for_sorting
from ietf.meeting.utils import session_requested_by
from ietf.meeting.utils import current_session_status
from ietf.meeting.utils import data_for_meetings_overview
from ietf.meeting.utils import session_requested_by, SaveMaterialsError
from ietf.meeting.utils import current_session_status, get_meeting_sessions, SessionNotScheduledError
from ietf.meeting.utils import data_for_meetings_overview, handle_upload_file, save_session_minutes_revision
from ietf.meeting.utils import preprocess_constraints_for_meeting_schedule_editor
from ietf.meeting.utils import diff_meeting_schedules, prefetch_schedule_diff_objects
from ietf.meeting.utils import swap_meeting_schedule_timeslot_assignments, bulk_create_timeslots
from ietf.meeting.utils import preprocess_meeting_important_dates
from ietf.message.utils import infer_message
from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName
from ietf.secr.proceedings.utils import handle_upload_file
from ietf.secr.proceedings.proc_utils import (get_progress_stats, post_process, import_audio_files,
create_recording)
from ietf.utils import markdown
from ietf.utils.decorators import require_api_key
from ietf.utils.hedgedoc import Note, NoteError
from ietf.utils.history import find_history_replacements_active_at
from ietf.utils.log import assertion
from ietf.utils.mail import send_mail_message, send_mail_text
@ -259,7 +259,7 @@ def materials_document(request, document, num=None, ext=None):
content_type = content_type.replace('plain', 'markdown', 1)
break;
elif atype[0] == 'text/html':
bytes = "<html>\n<head></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode(),extensions=['extra'])
bytes = "<html>\n<head></head>\n<body>\n%s\n</body>\n</html>\n" % markdown.markdown(bytes.decode())
content_type = content_type.replace('plain', 'html', 1)
break;
elif atype[0] == 'text/plain':
@ -2201,16 +2201,13 @@ def meeting_requests(request, num=None):
{"meeting": meeting, "sessions":sessions,
"groups_not_meeting": groups_not_meeting})
def get_sessions(num, acronym):
meeting = get_meeting(num=num,type_in=None)
sessions = Session.objects.filter(meeting=meeting,group__acronym=acronym,type__in=['regular','plenary','other'])
return sorted(
get_meeting_sessions(num, acronym).with_current_status(),
key=lambda s: session_time_for_sorting(s, use_meeting_date=False)
)
if not sessions:
sessions = Session.objects.filter(meeting=meeting,short=acronym,type__in=['regular','plenary','other'])
sessions = sessions.with_current_status()
return sorted(sessions, key=lambda s: session_time_for_sorting(s, use_meeting_date=False))
def session_details(request, num, acronym):
meeting = get_meeting(num=num,type_in=None)
@ -2367,13 +2364,14 @@ def upload_session_bluesheets(request, session_id, num):
ota = session.official_timeslotassignment()
sess_time = ota and ota.timeslot.time
if not sess_time:
return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain")
return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain")
save_error = save_bluesheet(request, session, file, encoding=form.file_encoding[file.name])
if save_error:
form.add_error(None, save_error)
else:
messages.success(request, 'Successfully uploaded bluesheets.')
return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
else:
form = UploadBlueSheetForm()
@ -2455,62 +2453,29 @@ def upload_session_minutes(request, session_id, num):
apply_to_all = session.type_id == 'regular'
if show_apply_to_all_checkbox:
apply_to_all = form.cleaned_data['apply_to_all']
if minutes_sp:
doc = minutes_sp.document
doc.rev = '%02d' % (int(doc.rev)+1)
minutes_sp.rev = doc.rev
minutes_sp.save()
# Set up the new revision
try:
save_session_minutes_revision(
session=session,
apply_to_all=apply_to_all,
file=file,
ext=ext,
encoding=form.file_encoding[file.name],
request=request,
)
except SessionNotScheduledError:
return HttpResponseGone(
"Cannot receive uploads for an unscheduled session. Please check the session ID.",
content_type="text/plain",
)
except SaveMaterialsError as err:
form.add_error(None, str(err))
else:
ota = session.official_timeslotassignment()
sess_time = ota and ota.timeslot.time
if not sess_time:
return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain")
if session.meeting.type_id=='ietf':
name = 'minutes-%s-%s' % (session.meeting.number,
session.group.acronym)
title = 'Minutes IETF%s: %s' % (session.meeting.number,
session.group.acronym)
if not apply_to_all:
name += '-%s' % (sess_time.strftime("%Y%m%d%H%M"),)
title += ': %s' % (sess_time.strftime("%a %H:%M"),)
else:
name = 'minutes-%s-%s' % (session.meeting.number, sess_time.strftime("%Y%m%d%H%M"))
title = 'Minutes %s: %s' % (session.meeting.number, sess_time.strftime("%a %H:%M"))
if Document.objects.filter(name=name).exists():
doc = Document.objects.get(name=name)
doc.rev = '%02d' % (int(doc.rev)+1)
else:
doc = Document.objects.create(
name = name,
type_id = 'minutes',
title = title,
group = session.group,
rev = '00',
)
DocAlias.objects.create(name=doc.name).docs.add(doc)
doc.states.add(State.objects.get(type_id='minutes',slug='active'))
if session.sessionpresentation_set.filter(document=doc).exists():
sp = session.sessionpresentation_set.get(document=doc)
sp.rev = doc.rev
sp.save()
else:
session.sessionpresentation_set.create(document=doc,rev=doc.rev)
if apply_to_all:
for other_session in sessions:
if other_session != session:
other_session.sessionpresentation_set.filter(document__type='minutes').delete()
other_session.sessionpresentation_set.create(document=doc,rev=doc.rev)
filename = '%s-%s%s'% ( doc.name, doc.rev, ext)
doc.uploaded_filename = filename
e = NewRevisionDocEvent.objects.create(doc=doc, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev, rev=doc.rev)
# The way this function builds the filename it will never trigger the file delete in handle_file_upload.
save_error = handle_upload_file(file, filename, session.meeting, 'minutes', request=request, encoding=form.file_encoding[file.name])
if save_error:
form.add_error(None, save_error)
else:
doc.save_with_history([e])
return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
else:
# no exception -- success!
messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.')
return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
else:
form = UploadMinutesForm(show_apply_to_all_checkbox)
return render(request, "meeting/upload_session_minutes.html",
@ -2532,7 +2497,7 @@ def upload_session_agenda(request, session_id, num):
session_number = None
sessions = get_sessions(session.meeting.number,session.group.acronym)
show_apply_to_all_checkbox = len(sessions) > 1 if session.type_id == 'regular' else False
show_apply_to_all_checkbox = len(sessions) > 1 if session.type.slug == 'regular' else False
if len(sessions) > 1:
session_number = 1 + sessions.index(session)
@ -2543,7 +2508,7 @@ def upload_session_agenda(request, session_id, num):
if form.is_valid():
file = request.FILES['file']
_, ext = os.path.splitext(file.name)
apply_to_all = session.type_id == 'regular'
apply_to_all = session.type.slug == 'regular'
if show_apply_to_all_checkbox:
apply_to_all = form.cleaned_data['apply_to_all']
if agenda_sp:
@ -2555,7 +2520,7 @@ def upload_session_agenda(request, session_id, num):
ota = session.official_timeslotassignment()
sess_time = ota and ota.timeslot.time
if not sess_time:
return HttpResponse("Cannot receive uploads for an unscheduled session. Please check the session ID.", status=410, content_type="text/plain")
return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain")
if session.meeting.type_id=='ietf':
name = 'agenda-%s-%s' % (session.meeting.number,
session.group.acronym)
@ -2603,6 +2568,7 @@ def upload_session_agenda(request, session_id, num):
form.add_error(None, save_error)
else:
doc.save_with_history([e])
messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.')
return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
else:
form = UploadAgendaForm(show_apply_to_all_checkbox, initial={'apply_to_all':session.type_id=='regular'})
@ -2698,6 +2664,9 @@ def upload_session_slides(request, session_id, num, name):
else:
doc.save_with_history([e])
post_process(doc)
messages.success(
request,
f'Successfully uploaded slides as revision {doc.rev} of {doc.name}.')
return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
else:
initial = {}
@ -2764,6 +2733,7 @@ def propose_session_slides(request, session_id, num):
msg.by = request.user.person
msg.save()
send_mail_message(request, msg)
messages.success(request, 'Successfully submitted proposed slides.')
return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym)
else:
initial = {}
@ -2787,6 +2757,7 @@ def remove_sessionpresentation(request, session_id, num, name):
c = DocEvent(type="added_comment", doc=sp.document, rev=sp.document.rev, by=request.user.person)
c.desc = "Removed from session: %s" % (session)
c.save()
messages.success(request, f'Successfully removed {name}.')
return redirect('ietf.meeting.views.session_details', num=session.meeting.number, acronym=session.group.acronym)
return render(request,'meeting/remove_sessionpresentation.html', {'sp': sp })
@ -4119,3 +4090,87 @@ def approve_proposed_slides(request, slidesubmission_id, num):
'existing_doc' : existing_doc,
'form': form,
})
def import_session_minutes(request, session_id, num):
"""Import session minutes from the ietf.notes.org site
A GET pulls in the markdown for a session's notes using the HedgeDoc API. An HTML preview of how
the datatracker will render the result is sent back. The confirmation form presented to the user
contains a hidden copy of the markdown source that will be submitted back if approved.
A POST accepts the hidden source and creates a new revision of the notes. This step does *not*
retrieve the note from the HedgeDoc API again - it posts the hidden source from the form. Any
changes to the HedgeDoc site after the preview was retrieved will be ignored. We could also pull
the source again and re-display the updated preview with an explanatory message, but there will
always be a race condition. Rather than add that complication, we assume that the user previewing
the imported minutes will be aware of anyone else changing the notes and coordinate with them.
A consequence is that the user could tamper with the hidden form and it would be accepted. This is
ok, though, because they could more simply upload whatever they want through the upload form with
the same effect so no exploit is introduced.
"""
session = get_object_or_404(Session, pk=session_id)
note = Note(session.notes_id())
if not session.can_manage_materials(request.user):
permission_denied(request, "You don't have permission to import minutes for this session.")
if session.is_material_submission_cutoff() and not has_role(request.user, "Secretariat"):
permission_denied(request, "The materials cutoff for this session has passed. Contact the secretariat for further action.")
if request.method == 'POST':
form = ImportMinutesForm(request.POST)
if not form.is_valid():
import_contents = form.data['markdown_text']
else:
import_contents = form.cleaned_data['markdown_text']
try:
save_session_minutes_revision(
session=session,
file=io.BytesIO(import_contents.encode('utf8')),
ext='.md',
request=request,
)
except SessionNotScheduledError:
return HttpResponseGone(
"Cannot import minutes for an unscheduled session. Please check the session ID.",
content_type="text/plain",
)
except SaveMaterialsError as err:
form.add_error(None, str(err))
else:
messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.')
return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
else:
try:
import_contents = note.get_source()
except NoteError as err:
messages.error(request, f'Could not import notes with id {note.id}: {err}.')
return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym)
form = ImportMinutesForm(initial={'markdown_text': import_contents})
# Try to prevent pointless revision creation. Note that we do not block replacing
# a document with an identical copy in the validation above. We cannot entirely
# avoid a race condition and the likelihood/amount of damage is very low so no
# need to complicate things further.
current_minutes = session.minutes()
contents_changed = True
if current_minutes:
try:
with open(current_minutes.get_file_name()) as f:
if import_contents == Note.preprocess_source(f.read()):
contents_changed = False
messages.warning(request, 'This document is identical to the current revision, no need to import.')
except FileNotFoundError:
pass # allow import if the file is missing
return render(
request,
'meeting/import_minutes.html',
{
'form': form,
'note': note,
'session': session,
'contents_changed': contents_changed,
},
)

2
ietf/meeting/views_proceedings.py
View file

@ -14,7 +14,7 @@ from ietf.meeting.forms import FileUploadForm
from ietf.meeting.models import Meeting, MeetingHost
from ietf.meeting.helpers import get_meeting
from ietf.name.models import ProceedingsMaterialTypeName
from ietf.secr.proceedings.utils import handle_upload_file
from ietf.meeting.utils import handle_upload_file
from ietf.utils.text import xslugify
class UploadProceedingsMaterialForm(FileUploadForm):

3
ietf/secr/meetings/views.py
View file

@ -20,14 +20,13 @@ from ietf.utils.mail import send_mail
from ietf.meeting.forms import duration_string
from ietf.meeting.helpers import get_meeting, make_materials_directories, populate_important_dates
from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent
from ietf.meeting.utils import add_event_info_to_session_qs
from ietf.meeting.utils import add_event_info_to_session_qs, handle_upload_file
from ietf.name.models import SessionStatusName
from ietf.group.models import Group, GroupEvent
from ietf.secr.meetings.blue_sheets import create_blue_sheets
from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm,
MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm,
UploadBlueSheetForm, MeetingRoomOptionsForm )
from ietf.secr.proceedings.utils import handle_upload_file
from ietf.secr.sreq.views import get_initial_session
from ietf.secr.utils.meeting import get_session, get_timeslot
from ietf.mailtrigger.utils import gather_address_lists

74
ietf/secr/proceedings/utils.py
View file

@ -1,74 +0,0 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
import glob
import io
import os
from django.conf import settings
from django.contrib import messages
from django.utils.encoding import smart_text
import debug # pyflakes:ignore
from ietf.utils.html import sanitize_document
def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None):
'''
This function takes a file object, a filename and a meeting object and subdir as string.
It saves the file to the appropriate directory, get_materials_path() + subdir.
If the file is a zip file, it creates a new directory in 'slides', which is the basename of the
zip file and unzips the file in the new directory.
'''
base, extension = os.path.splitext(filename)
if extension == '.zip':
path = os.path.join(meeting.get_materials_path(),subdir,base)
if not os.path.exists(path):
os.mkdir(path)
else:
path = os.path.join(meeting.get_materials_path(),subdir)
if not os.path.exists(path):
os.makedirs(path)
# agendas and minutes can only have one file instance so delete file if it already exists
if subdir in ('agenda','minutes'):
old_files = glob.glob(os.path.join(path,base) + '.*')
for f in old_files:
os.remove(f)
destination = io.open(os.path.join(path,filename), 'wb+')
if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
file.open()
text = file.read()
if encoding:
try:
text = text.decode(encoding)
except LookupError as e:
return "Failure trying to save '%s': Could not identify the file encoding, got '%s'. Hint: Try to upload as UTF-8." % (filename, str(e)[:120])
else:
try:
text = smart_text(text)
except UnicodeDecodeError as e:
return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120])
# Whole file sanitization; add back what's missing from a complete
# document (sanitize will remove these).
clean = sanitize_document(text)
destination.write(clean.encode('utf8'))
if request and clean != text:
messages.warning(request, "Uploaded html content is sanitized to prevent unsafe content. "
"Your upload %s was changed by the sanitization; please check the "
"resulting content. " % (filename, ))
else:
if hasattr(file, 'chunks'):
for chunk in file.chunks():
destination.write(chunk)
else:
destination.write(file.read())
destination.close()
# unzip zipfile
if extension == '.zip':
os.chdir(path)
os.system('unzip %s' % filename)
return None

2
ietf/settings.py
View file

@ -148,6 +148,7 @@ IETF_ID_URL = IETF_HOST_URL + 'id/' # currently unused
IETF_ID_ARCHIVE_URL = IETF_HOST_URL + 'archive/id/'
IETF_AUDIO_URL = IETF_HOST_URL + 'audio/'
IETF_NOTES_URL = 'https://notes.ietf.org/' # HedgeDoc base URL
# Absolute path to the directory static files should be collected to.
# Example: "/var/www/example.com/static/"
@ -996,6 +997,7 @@ DOT_BINARY = '/usr/bin/dot'
UNFLATTEN_BINARY= '/usr/bin/unflatten'
RSYNC_BINARY = '/usr/bin/rsync'
YANGLINT_BINARY = '/usr/bin/yanglint'
DE_GFM_BINARY = '/usr/bin/de-gfm.ruby2.5'
# Account settings
DAYS_TO_EXPIRE_REGISTRATION_LINK = 3

36
ietf/templates/meeting/import_minutes.html Normal file
View file

@ -0,0 +1,36 @@
{% extends "base.html" %}
{% load bootstrap3 origin %}
{% block morecss %}
#preview { width: 100%; height: 60vh; border: solid 2px; }
{% endblock %}
{% block title %}
Import Preview: {% firstof note.get_title note.id %}
{% endblock %}
{% block content %}
{% origin %}
<h1>Import Preview: {% firstof note.get_title note.id %}</h1>
<div class="row">
<div class="col-md-11">
<p class="pull-left">Last updated: {% firstof note.get_update_time "unknown" %}</p>
<p class="pull-right"><a href="{{ note.url }}">View on notes.ietf.org</a></p>
<iframe id="preview" srcdoc="{{ note.get_preview|force_escape }}" sandbox>
<p>Your browser does not support iframes, so preview cannot be shown.</p>
</iframe>
</div>
</div>
<div class="row">
<div class="col-md-11">
<form method="post" enctype="multipart/form-data">
{% csrf_token %}
{% bootstrap_form form %}
{% buttons %}
<button type="submit" class="btn btn-primary pull-left" {% if not contents_changed %}disabled{% endif %}> Import </button>
<a class="btn btn-default pull-right" href="{% url 'ietf.meeting.views.session_details' num=session.meeting.number acronym=session.group.acronym %}"> Back </a>
{% endbuttons %}
</form>
</div>
</div>
{% endblock %}

6
ietf/templates/meeting/interim_session_buttons.html
View file

@ -16,11 +16,7 @@
{% endif %}
<!-- etherpad -->
{% if use_codimd %}
{% if item.slot_type.slug == 'plenary' %}
<a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-plenary" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% else %}
<a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-{{acronym}}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% endif %}
<a class="" href="{{ session.notes_url }}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% endif %}
{# show stream buttons up till end of session, then show archive buttons #}

6
ietf/templates/meeting/session_buttons_include.html
View file

@ -22,11 +22,7 @@
<!-- HedgeDoc -->
{% if use_codimd %}
{% if timeslot.type.slug == 'plenary' %}
<a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-plenary" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% else %}
<a class="" href="https://notes.ietf.org/notes-ietf-{{ meeting.number }}-{{acronym}}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% endif %}
<a class="" href="{{ session.notes_url }}" title="Notepad for note-takers"><span class="fa fa-fw fa-edit"></span></a>
{% endif %}
{# show stream buttons up till end of session, then show archive buttons #}

3
ietf/templates/meeting/session_details_panel.html
View file

@ -74,6 +74,9 @@
{% endif %}
{% if pres.document.type.slug != 'bluesheets' or user|has_role:"Secretariat" or meeting.type.slug == 'interim' and can_manage_materials %}
<a class="btn btn-default btn-sm pull-right" href="{{upload_url}}">Upload Revision</a>
{% if pres.document.type.slug == 'minutes' %}
<a class="btn btn-default btn-sm pull-right" href="{% url 'ietf.meeting.views.import_session_minutes' num=session.meeting.number session_id=session.pk %}">Import from notes.ietf.org</a>
{% endif %}
{% endif %}
</td>
{% endif %}

117
ietf/utils/hedgedoc.py Normal file
View file

@ -0,0 +1,117 @@
# Copyright The IETF Trust 2021, All Rights Reserved
# -*- coding: utf-8 -*-
"""HedgeDoc API utilities"""
import json
import requests
import subprocess
import debug # pyflakes: ignore
from urllib.parse import urljoin
from django.conf import settings
from ietf.utils.markdown import markdown
class Note:
base_url = settings.IETF_NOTES_URL
def __init__(self, id):
self.id = id
self.url = urljoin(self.base_url, self.id) # URL on notes site
self._metadata = None
self._preview = None
self._source = None
@classmethod
def preprocess_source(cls, raw_source):
"""Perform preprocessing on raw source input
Guaranteed to process input in the same way that the Note class processes
markdown source pulled from the notes site.
"""
return de_gfm(raw_source)
def get_source(self):
"""Retrieve markdown source from hedgedoc
Converts line breaks from GitHub Flavored Markdown (GFM) style to
to traditional markdown.
"""
if self._source is None:
try:
r = requests.get(urljoin(self.base_url, f'{self.id}/download'), allow_redirects=True)
except requests.RequestException:
raise ServerNoteError
if r.status_code != 200:
raise NoteNotFound
self._source = self.preprocess_source(r.text)
return self._source
def get_preview(self):
if self._preview is None:
self._preview = markdown(self.get_source())
return self._preview
def get_title(self):
try:
metadata = self._retrieve_metadata()
except NoteError:
metadata = {} # don't let an error retrieving the title prevent retrieval
return metadata.get('title', None)
def get_update_time(self):
try:
metadata = self._retrieve_metadata()
except NoteError:
metadata = {} # don't let an error retrieving the update timestamp prevent retrieval
return metadata.get('updatetime', None)
def _retrieve_metadata(self):
if self._metadata is None:
try:
r = requests.get(urljoin(self.base_url, f'{self.id}/info'), allow_redirects=True)
except requests.RequestException:
raise ServerNoteError
if r.status_code != 200:
raise NoteNotFound
try:
self._metadata = json.loads(r.content)
except json.JSONDecodeError:
raise InvalidNote
return self._metadata
def de_gfm(source: str):
"""Convert GFM line breaks to standard Markdown
Calls de-gfm from the kramdown-rfc2629 gem.
"""
result = subprocess.run(
[settings.DE_GFM_BINARY,],
stdout=subprocess.PIPE, # post-Python 3.7, this can be replaced with capture_output=True
input=source,
encoding='utf8',
check=True,
)
return result.stdout
class NoteError(Exception):
"""Base class for exceptions in this module"""
default_message = 'A note-related error occurred'
def __init__(self, *args, **kwargs):
if not args:
args = (self.default_message, )
super().__init__(*args)
class ServerNoteError(NoteError):
default_message = 'Could not reach the notes server'
class NoteNotFound(NoteError):
default_message = 'Note did not exist or could not be loaded'
class InvalidNote(NoteError):
default_message = 'Note data invalid'

20
ietf/utils/markdown.py Normal file
View file

@ -0,0 +1,20 @@
# Copyright The IETF Trust 2021, All Rights Reserved
# -*- coding: utf-8 -*-
"""Markdown wrapper
Use this instead of importing markdown directly to guarantee consistent extensions / options through
the datatracker.
"""
import bleach
import markdown as python_markdown
from django.utils.safestring import mark_safe
from markdown.extensions.extra import ExtraExtension
ALLOWED_TAGS = bleach.ALLOWED_TAGS + ['p', 'h1', 'h2', 'h3', 'h4', 'br']
def markdown(text):
return mark_safe(bleach.clean(
python_markdown.markdown(text, extensions=[ExtraExtension()]),
tags=ALLOWED_TAGS,
))

18
ietf/utils/test_utils.py
View file

@ -38,6 +38,7 @@ import os
import re
import email
import html5lib
import requests_mock
import shutil
import sys
@ -151,6 +152,7 @@ class ReverseLazyTest(django.test.TestCase):
response = self.client.get('/ipr/update/')
self.assertRedirects(response, "/ipr/", status_code=301)
class TestCase(django.test.TestCase):
"""IETF TestCase class
@ -158,6 +160,7 @@ class TestCase(django.test.TestCase):
* asserts for html5 validation.
* tempdir() convenience method
* setUp() and tearDown() that override settings paths with temp directories
* mocking the requests library to prevent dependencies on the outside network
The setUp() and tearDown() methods create / remove temporary paths and override
Django's settings with the temp dir names. Subclasses of this class must
@ -165,6 +168,12 @@ class TestCase(django.test.TestCase):
anew for each test to avoid risk of cross-talk between test cases. Overriding
the settings_temp_path_overrides class value will modify which path settings are
replaced with temp test dirs.
Uses requests-mock to prevent the requests library from making requests to outside
resources. The requests-mock library allows nested mocks, so individual tests can
ignore this. Note that the mock set up by this class will intercept any requests
not handled by a test's inner mock - even if the latter is created with
real_http=True.
"""
# These settings will be overridden with empty temporary directories
settings_temp_path_overrides = [
@ -257,10 +266,14 @@ class TestCase(django.test.TestCase):
def __str__(self):
return u"%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName)
def setUp(self):
# Replace settings paths with temporary directories.
super().setUp()
# Prevent the requests library from making live requests during tests
self.requests_mock = requests_mock.Mocker()
self.requests_mock.start()
# Replace settings paths with temporary directories.
self._ietf_temp_dirs = {} # trashed during tearDown, DO NOT put paths you care about in this
for setting in self.settings_temp_path_overrides:
self._ietf_temp_dirs[setting] = self.tempdir(slugify(setting))
@ -271,4 +284,5 @@ class TestCase(django.test.TestCase):
self._ietf_saved_context.disable()
for dir in self._ietf_temp_dirs.values():
shutil.rmtree(dir)
self.requests_mock.stop()
super().tearDown()

45
ietf/utils/tests_hedgedoc.py Normal file
View file

@ -0,0 +1,45 @@
# Copyright The IETF Trust 2021, All Rights Reserved
# -*- coding: utf-8 -*-
"""HedgeDoc API utilities tests"""
import requests_mock
from ietf.utils.tests import TestCase
from ietf.utils.hedgedoc import Note
class NoteTests(TestCase):
SAMPLE_MARKDOWN = ''.join((
'# Standard Markdown\n',
'This is a small sample of markdown text. It uses GFM-style line breaks.\n',
'\n',
'This is a second paragraph.\n',
'It has line breaks in GFM style.\n',
'And also some standard line breaks. \n',
'That is all.\n',
))
SAMPLE_MARKDOWN_OUTPUT = ''.join((
'# Standard Markdown {#standard-markdown}\n',
'\n',
'This is a small sample of markdown text. It uses GFM-style line breaks.\n',
'\n',
'This is a second paragraph. \n',
'It has line breaks in GFM style. \n',
'And also some standard line breaks. \n',
'That is all.\n',
'\n',
))
def test_retrieves_note(self):
with requests_mock.Mocker() as mock:
mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN)
n = Note('my_id')
result = n.get_source()
self.assertEqual(result, self.SAMPLE_MARKDOWN_OUTPUT)
def test_uses_preprocess_class_method(self):
"""Imported text should be processed by the preprocess_source class method"""
with requests_mock.Mocker() as mock:
mock.get('https://notes.ietf.org/my_id/download', text=self.SAMPLE_MARKDOWN)
n = Note('my_id')
result = n.get_source()
self.assertEqual(result, Note.preprocess_source(self.SAMPLE_MARKDOWN))