From 25cc00f925ad49be2515450e7652e2876382d05c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 2 Jan 2019 22:55:00 +0000 Subject: [PATCH] Refactored DocumentInfo to address overloading the external_url field with strings that are not URLs. Commit ready for merge. - Legacy-Id: 15864 --- .../migrations/0008_add_uploaded_filename.py | 25 ++++++++++ ...n_url_externalurls_to_uploaded_filename.py | 46 +++++++++++++++++++ ietf/doc/models.py | 33 ++++++------- ietf/doc/resources.py | 2 + ietf/doc/views_doc.py | 8 ++-- ietf/liaisons/forms.py | 2 +- ietf/liaisons/tests.py | 6 +-- ietf/meeting/feeds.py | 4 +- ietf/meeting/forms.py | 4 +- ietf/meeting/helpers.py | 3 +- ietf/meeting/models.py | 8 ++-- ietf/meeting/test_data.py | 14 +++--- ietf/meeting/tests_views.py | 8 ++-- ietf/meeting/views.py | 12 ++--- ietf/secr/proceedings/proc_utils.py | 6 +-- ietf/secr/proceedings/views.py | 17 ++++--- ietf/secr/utils/document.py | 4 +- ietf/settings.py | 10 ++-- 18 files changed, 143 insertions(+), 69 deletions(-) create mode 100644 ietf/doc/migrations/0008_add_uploaded_filename.py create mode 100644 ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py diff --git a/ietf/doc/migrations/0008_add_uploaded_filename.py b/ietf/doc/migrations/0008_add_uploaded_filename.py new file mode 100644 index 000000000..769139f42 --- /dev/null +++ b/ietf/doc/migrations/0008_add_uploaded_filename.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2018-12-28 13:11 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0007_idexists'), + ] + + operations = [ + migrations.AddField( + model_name='dochistory', + name='uploaded_filename', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='document', + name='uploaded_filename', + field=models.TextField(blank=True), + ), + ] diff --git a/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py b/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py new file mode 100644 index 000000000..2d014e1db --- /dev/null +++ b/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.17 on 2018-12-28 13:33 +from __future__ import unicode_literals + +from django.db import migrations +from django.db.models import F + +def forward(apps, schema_editor): + Document = apps.get_model('doc','Document') + Document.objects.exclude(type_id__in=('review','recording')).update(uploaded_filename = F('external_url')) + Document.objects.exclude(type_id__in=('review','recording')).update(external_url="") + + Document.objects.filter(name='slides-100-edu-sessf-patents-at-ietf-an-overview-of-bcp79bis').update(uploaded_filename='slides-100-edu-sessf-patents-at-ietf-an-overview-of-bcp79bis-00.pdf') + + DocHistory = apps.get_model('doc','DocHistory') + DocHistory.objects.exclude(type_id__in=('review','recording')).update(uploaded_filename = F('external_url')) + DocHistory.objects.exclude(type_id__in=('review','recording')).update(external_url="") + + DocHistory.objects.filter(uploaded_filename='https://www.ietf.org/proceedings/97/slides/slides-97-edu-sessb-local-version-of-newcomers-training-in-korean-00.pdf').update(uploaded_filename='slides-97-edu-sessb-local-version-of-newcomers-training-in-korean-00.pdf') + DocHistory.objects.filter(uploaded_filename='http://materials-98-codec-opus-newvectors-00.tar.gz').update(uploaded_filename='materials-98-codec-opus-newvectors-00.tar.gz') + DocHistory.objects.filter(uploaded_filename='http://materials-98-codec-opus-update-00.patch').update(uploaded_filename='materials-98-codec-opus-update-00.patch') + DocHistory.objects.filter(uploaded_filename='http://slides-100-edu-sessf-patents-at-ietf-an-overview-of-bcp79bis-00.pdf').update(uploaded_filename='slides-100-edu-sessf-patents-at-ietf-an-overview-of-bcp79bis-00.pdf') + DocHistory.objects.filter(uploaded_filename='http://bluesheets-97-6man-201611150930-00.pdf/').update(uploaded_filename='bluesheets-97-6man-201611150930-00.pdf') + DocHistory.objects.filter(uploaded_filename='http://agenda-interim-2017-stir-01-stir-01-01.txt').update(uploaded_filename='agenda-interim-2017-stir-01-stir-01-01.txt') + DocHistory.objects.filter(uploaded_filename='http://agenda-interim-2017-icnrg-02-icnrg-01-05.html').update(uploaded_filename='agenda-interim-2017-icnrg-02-icnrg-01-05.html') + + +def reverse(apps, schema_editor): + Document = apps.get_model('doc','Document') + Document.objects.exclude(type_id__in=('review','recording')).update(external_url = F('uploaded_filename')) + Document.objects.exclude(type_id__in=('review','recording')).update(uploaded_filename="") + + DocHistory = apps.get_model('doc','DocHistory') + DocHistory.objects.exclude(type_id__in=('review','recording')).update(external_url = F('uploaded_filename')) + DocHistory.objects.exclude(type_id__in=('review','recording')).update(uploaded_filename="") + + +class Migration(migrations.Migration): + + dependencies = [ + ('doc', '0008_add_uploaded_filename'), + ] + + operations = [ + migrations.RunPython(forward,reverse), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index ce398c251..8fa3706d5 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -97,13 +97,19 @@ class DocumentInfo(models.Model): shepherd = ForeignKey(Email, related_name='shepherd_%(class)s_set', blank=True, null=True) expires = models.DateTimeField(blank=True, null=True) notify = models.CharField(max_length=255, blank=True) - external_url = models.URLField(blank=True) # Should be set for documents with type 'External'. + external_url = models.URLField(blank=True) + uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) internal_comments = models.TextField(blank=True) def file_extension(self): - _,ext = os.path.splitext(self.external_url) - return ext.lstrip(".").lower() + if not hasattr(self, '_cached_extension'): + if self.uploaded_filename: + _, ext= os.path.splitext(self.uploaded_filename) + self._cached_extension = ext.lstrip(".").lower() + else: + self._cached_extension = "txt" + return self._cached_extension def get_file_path(self): if not hasattr(self, '_cached_file_path'): @@ -138,7 +144,9 @@ class DocumentInfo(models.Model): def get_base_name(self): if not hasattr(self, '_cached_base_name'): - if self.type_id == 'draft': + if self.uploaded_filename: + self._cached_base_name = self.uploaded_filename + elif self.type_id == 'draft': if self.is_dochistory(): self._cached_base_name = "%s-%s.txt" % (self.doc.name, self.rev) else: @@ -147,12 +155,9 @@ class DocumentInfo(models.Model): else: self._cached_base_name = "%s-%s.txt" % (self.name, self.rev) elif self.type_id in ["slides", "agenda", "minutes", "bluesheets", ] and self.meeting_related(): - if self.external_url: - # we need to remove the extension for the globbing below to work - self._cached_base_name = self.external_url - else: - self._cached_base_name = "%s.txt" % self.canonical_name() # meeting materials are unversioned at the moment + self._cached_base_name = "%s-%s.txt" % self.canonical_name() elif self.type_id == 'review': + # TODO: This will be wrong if a review is updated on the same day it was created (or updated more than once on the same day) self._cached_base_name = "%s.txt" % self.name else: if self.rev: @@ -195,8 +200,6 @@ class DocumentInfo(models.Model): # the earlier resulution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - # XXX TODO: move all non-URL 'external_url' values into a field which is - # better named, or regularize the filename based on self.name if not hasattr(self, '_cached_href'): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: @@ -204,6 +207,7 @@ class DocumentInfo(models.Model): validator(self.external_url) return self.external_url except ValidationError: + log.unreachable('2018-12-28') pass @@ -223,10 +227,7 @@ class DocumentInfo(models.Model): elif self.type_id in meeting_doc_refs: self.is_meeting_related = True else: - if len(self.external_url): - return self.external_url - else: - return None + return None if self.is_meeting_related: if not meeting: @@ -625,7 +626,7 @@ class Document(DocumentInfo): if self.type_id == 'recording': url = self.external_url else: - filename = self.external_url + filename = self.uploaded_filename url = '%sproceedings/%s/%s/%s' % (settings.IETF_HOST_URL,meeting.number,self.type_id,filename) else: url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ 'name': name }, urlconf="ietf.urls") diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 71f073b47..62f928ced 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -116,6 +116,7 @@ class DocumentResource(ModelResource): "expires": ALL, "notify": ALL, "external_url": ALL, + "uploaded_filename": ALL, "note": ALL, "internal_comments": ALL, "name": ALL, @@ -228,6 +229,7 @@ class DocHistoryResource(ModelResource): "expires": ALL, "notify": ALL, "external_url": ALL, + "uploaded_filename": ALL, "note": ALL, "internal_comments": ALL, "name": ALL, diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index e67d753c1..84e4a6b68 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -549,11 +549,9 @@ def document_main(request, name, rev=None): if doc.type_id in ("slides", "agenda", "minutes", "bluesheets",): can_manage_material = can_manage_materials(request.user, doc.group) presentations = doc.future_presentations() - if doc.meeting_related(): - basename = doc.canonical_name() # meeting materials are unversioned at the moment - if doc.external_url: - # we need to remove the extension for the globbing below to work - basename = os.path.splitext(doc.external_url)[0] + if doc.uploaded_filename: + # we need to remove the extension for the globbing below to work + basename = os.path.splitext(doc.uploaded_filename)[0] else: basename = "%s-%s" % (doc.canonical_name(), doc.rev) diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 4eae92aae..b059ed5f2 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -366,7 +366,7 @@ class LiaisonModelForm(BetterModelForm): defaults=dict( title = attachment_title, type_id = "liai-att", - external_url = name + extension, # strictly speaking not necessary, but just for the time being ... + uploaded_filename = name + extension, ) ) if created: diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index cbecd3fe2..fa2a1bfb2 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -432,7 +432,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(new_liaison.attachments.count(), attachments_before + 1) attachment = new_liaison.attachments.order_by("-name")[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.external_url)) as f: + with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -736,7 +736,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.external_url)) as f: + with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -815,7 +815,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.external_url)) as f: + with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) diff --git a/ietf/meeting/feeds.py b/ietf/meeting/feeds.py index 1366da74f..a2780065d 100644 --- a/ietf/meeting/feeds.py +++ b/ietf/meeting/feeds.py @@ -15,12 +15,14 @@ class LatestMeetingMaterialFeed(Feed): def items(self): objs = [] + # FIXME: why aren't other materials types in here? for doc in Document.objects.filter(type__in=("agenda", "minutes", "slides")).order_by('-time')[:60]: obj = dict( title=doc.type_id, group_acronym=doc.name.split("-")[2], date=doc.time, - link=self.base_url + os.path.join(doc.get_file_path(), doc.external_url)[len(settings.AGENDA_PATH):], + # FIXME: why isn't this using gref or href? + link=self.base_url + os.path.join(doc.get_file_path(), doc.uploaded_filename)[len(settings.AGENDA_PATH):], author="" ) objs.append(obj) diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index 75b97fd08..645ad6b40 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -267,7 +267,9 @@ class InterimSessionModelForm(forms.ModelForm): group=self.group, name=filename, rev='00', - external_url='{}-00.txt'.format(filename)) + # FIXME: if these are always computed, they shouldn't be in uploaded_filename - just compute them when needed + # FIXME: What about agendas in html or markdown format? + uploaded_filename='{}-00.txt'.format(filename)) doc.set_state(State.objects.get(type__slug=doc.type.slug, slug='active')) DocAlias.objects.create(name=doc.name, document=doc) self.instance.sessionpresentation_set.create(document=doc, rev=doc.rev) diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py index ead482bf9..ee6b5cff7 100644 --- a/ietf/meeting/helpers.py +++ b/ietf/meeting/helpers.py @@ -208,7 +208,8 @@ def read_session_file(type, num, doc): # style python format, something like this: # DOC_PATH_FORMAT = { "agenda": "/foo/bar/agenda-{meeting.number}/agenda-{meeting-number}-{doc.group}*", } # - path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.external_url)) + # FIXME: uploaded_filename should be replaced with a function call that computes names that are fixed + path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.uploaded_filename)) if os.path.exists(path): with open(path) as f: return f.read(), path diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index f995713cc..f79c2f16e 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1107,7 +1107,7 @@ class Session(models.Model): def agenda_text(self): doc = self.agenda() if doc: - path = os.path.join(settings.AGENDA_PATH, self.meeting.number, "agenda", doc.external_url) + path = os.path.join(settings.AGENDA_PATH, self.meeting.number, "agenda", doc.uploaded_filename) if os.path.exists(path): with open(path) as f: return f.read() @@ -1130,10 +1130,8 @@ class Session(models.Model): if not agenda: return "" - # we use external_url at the moment, should probably regularize - # the filenames to match the document name instead - filename = agenda.external_url - self._agenda_file = "%s/agenda/%s" % (self.meeting.number, filename) + # FIXME: uploaded_filename should be replaced with a function that computes filenames when they are of a fixed schema and not uploaded names + self._agenda_file = "%s/agenda/%s" % (self.meeting.number, agenda.uploaded_filename) return self._agenda_file diff --git a/ietf/meeting/test_data.py b/ietf/meeting/test_data.py index 5f01f82fd..349f1fbe7 100644 --- a/ietf/meeting/test_data.py +++ b/ietf/meeting/test_data.py @@ -37,7 +37,7 @@ def make_interim_meeting(group,date,status='sched'): rev = '00' file = "%s-%s.txt" % (name, rev) doc = DocumentFactory.create(name=name, type_id='agenda', title="Agenda", - external_url=file, group=group, rev=rev, states=[('draft','active')]) + uploaded_filename=file, group=group, rev=rev, states=[('draft','active')]) pres = SessionPresentation.objects.create(session=session, document=doc, rev=doc.rev) session.sessionpresentation_set.add(pres) # minutes @@ -45,7 +45,7 @@ def make_interim_meeting(group,date,status='sched'): rev = '00' file = "%s-%s.txt" % (name, rev) doc = DocumentFactory.create(name=name, type_id='minutes', title="Minutes", - external_url=file, group=group, rev=rev, states=[('draft','active')]) + uploaded_filename=file, group=group, rev=rev, states=[('draft','active')]) pres = SessionPresentation.objects.create(session=session, document=doc, rev=doc.rev) session.sessionpresentation_set.add(pres) # slides @@ -55,7 +55,7 @@ def make_interim_meeting(group,date,status='sched'): rev = '00' file = "%s-%s.txt" % (name, rev) doc = DocumentFactory.create(name=name, type_id='slides', title=title, - external_url=file, group=group, rev=rev, + uploaded_filename=file, group=group, rev=rev, states=[('slides','active'), ('reuse_policy', 'single')]) pres = SessionPresentation.objects.create(session=session, document=doc, rev=doc.rev) session.sessionpresentation_set.add(pres) @@ -158,23 +158,23 @@ def make_meeting_test_data(meeting=None): doc = DocumentFactory.create(name='agenda-42-mars', type_id='agenda', title="Agenda", - external_url="agenda-42-mars.txt", group=mars, rev='00', states=[('draft','active')]) + uploaded_filename="agenda-42-mars.txt", group=mars, rev='00', states=[('draft','active')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) # doc = DocumentFactory.create(name='minutes-42-mars', type_id='minutes', title="Minutes", - external_url="minutes-42-mars.txt", group=mars, rev='00', states=[('minutes','active')]) + uploaded_filename="minutes-42-mars.txt", group=mars, rev='00', states=[('minutes','active')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) doc = DocumentFactory.create(name='slides-42-mars-1-active', type_id='slides', title="Slideshow", - external_url="slides-42-mars.txt", group=mars, rev='00', + uploaded_filename="slides-42-mars.txt", group=mars, rev='00', states=[('slides','active'), ('reuse_policy', 'single')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) doc = DocumentFactory.create(name='slides-42-mars-2-deleted', type_id='slides', - title="Bad Slideshow", external_url="slides-42-mars-2-deleted.txt", group=mars, rev='00', + title="Bad Slideshow", uploaded_filename="slides-42-mars-2-deleted.txt", group=mars, rev='00', states=[('slides','deleted'), ('reuse_policy', 'single')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 4598065ab..6ae54e51b 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -51,7 +51,7 @@ class MeetingTests(TestCase): shutil.rmtree(self.materials_dir) def write_materials_file(self, meeting, doc, content): - path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, doc.external_url)) + path = os.path.join(self.materials_dir, "%s/%s/%s" % (meeting.number, doc.type_id, doc.uploaded_filename)) dirname = os.path.dirname(path) if not os.path.exists(dirname): @@ -158,9 +158,9 @@ class MeetingTests(TestCase): self.assertTrue(session.group.parent.acronym.upper() in agenda_content) self.assertTrue(slot.location.name in agenda_content) - self.assertTrue(session.materials.get(type='agenda').external_url in unicontent(r)) - self.assertTrue(session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().external_url in unicontent(r)) - self.assertFalse(session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().external_url in unicontent(r)) + self.assertTrue(session.materials.get(type='agenda').uploaded_filename in unicontent(r)) + self.assertTrue(session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().uploaded_filename in unicontent(r)) + self.assertFalse(session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().uploaded_filename in unicontent(r)) # iCal r = self.client.get(urlreverse("ietf.meeting.views.ical_agenda", kwargs=dict(num=meeting.number)) diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index c1d33e51b..bffbebc17 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -530,12 +530,12 @@ def agenda_csv(schedule, filtered_assignments): def agenda_field(item): agenda_doc = item.session.agenda() if agenda_doc: - return "http://www.ietf.org/proceedings/{schedule.meeting.number}/agenda/{agenda.external_url}".format(schedule=schedule, agenda=agenda_doc) + return "http://www.ietf.org/proceedings/{schedule.meeting.number}/agenda/{agenda.uploaded_filename}".format(schedule=schedule, agenda=agenda_doc) else: return "" def slides_field(item): - return "|".join("http://www.ietf.org/proceedings/{schedule.meeting.number}/slides/{slide.external_url}".format(schedule=schedule, slide=slide) for slide in item.session.slides()) + return "|".join("http://www.ietf.org/proceedings/{schedule.meeting.number}/slides/{slide.uploaded_filename}".format(schedule=schedule, slide=slide) for slide in item.session.slides()) write_row(headings) @@ -1210,7 +1210,7 @@ def upload_session_bluesheets(request, session_id, num): doc.docalias_set.create(name=doc.name) session.sessionpresentation_set.create(document=doc,rev='00') filename = '%s-%s%s'% ( doc.name, doc.rev, ext) - doc.external_url = filename + doc.uploaded_filename = filename e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) save_error = handle_upload_file(file, filename, session.meeting, 'bluesheets', request=request, encoding=form.file_encoding[file.name]) if save_error: @@ -1310,7 +1310,7 @@ def upload_session_minutes(request, session_id, num): other_session.sessionpresentation_set.filter(document__type='minutes').delete() other_session.sessionpresentation_set.create(document=doc,rev=doc.rev) filename = '%s-%s%s'% ( doc.name, doc.rev, ext) - doc.external_url = filename + doc.uploaded_filename = filename e = NewRevisionDocEvent.objects.create(doc=doc, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev, rev=doc.rev) # The way this function builds the filename it will never trigger the file delete in handle_file_upload. save_error = handle_upload_file(file, filename, session.meeting, 'minutes', request=request, encoding=form.file_encoding[file.name]) @@ -1413,7 +1413,7 @@ def upload_session_agenda(request, session_id, num): other_session.sessionpresentation_set.filter(document__type='agenda').delete() other_session.sessionpresentation_set.create(document=doc,rev=doc.rev) filename = '%s-%s%s'% ( doc.name, doc.rev, ext) - doc.external_url = filename + doc.uploaded_filename = filename e = NewRevisionDocEvent.objects.create(doc=doc,by=request.user.person,type='new_revision',desc='New revision available: %s'%doc.rev,rev=doc.rev) # The way this function builds the filename it will never trigger the file delete in handle_file_upload. save_error = handle_upload_file(file, filename, session.meeting, 'agenda', request=request, encoding=form.file_encoding[file.name]) @@ -1518,7 +1518,7 @@ def upload_session_slides(request, session_id, num, name): max_order = other_session.sessionpresentation_set.filter(document__type='slides').aggregate(Max('order'))['order__max'] or 0 other_session.sessionpresentation_set.create(document=doc,rev=doc.rev,order=max_order+1) filename = '%s-%s%s'% ( doc.name, doc.rev, ext) - doc.external_url = filename + doc.uploaded_filename = filename e = NewRevisionDocEvent.objects.create(doc=doc,by=request.user.person,type='new_revision',desc='New revision available: %s'%doc.rev,rev=doc.rev) # The way this function builds the filename it will never trigger the file delete in handle_file_upload. save_error = handle_upload_file(file, filename, session.meeting, 'slides', request=request, encoding=form.file_encoding[file.name]) diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py index 22c8484fe..549d37237 100644 --- a/ietf/secr/proceedings/proc_utils.py +++ b/ietf/secr/proceedings/proc_utils.py @@ -271,14 +271,14 @@ def post_process(doc): try: cmd = settings.SECR_PPT2PDF_COMMAND cmd.append(doc.get_file_path()) # outdir - cmd.append(os.path.join(doc.get_file_path(),doc.external_url)) # filename + cmd.append(os.path.join(doc.get_file_path(),doc.uploaded_filename)) # filename subprocess.check_call(cmd) except (subprocess.CalledProcessError, OSError) as error: log("Error converting PPT: %s" % (error)) return # change extension - base,ext = os.path.splitext(doc.external_url) - doc.external_url = base + '.pdf' + base,ext = os.path.splitext(doc.uploaded_filename) + doc.uploaded_filename = base + '.pdf' e = DocEvent.objects.create( type='changed_document', diff --git a/ietf/secr/proceedings/views.py b/ietf/secr/proceedings/views.py index 299a4b03f..571d159c2 100644 --- a/ietf/secr/proceedings/views.py +++ b/ietf/secr/proceedings/views.py @@ -51,13 +51,12 @@ def find_index(slide_id, qs): def get_doc_filename(doc): ''' This function takes a Document of type slides,minute or agenda and returns - the full path to the file on disk. During migration of the system the - filename was saved in external_url, new files will also use this convention. + the full path to the file on disk. ''' session = doc.session_set.all()[0] meeting = session.meeting if doc.external_url: - return os.path.join(meeting.get_materials_path(),doc.type.slug,doc.external_url) + return os.path.join(meeting.get_materials_path(),doc.type.slug,doc.uploaded_filename) else: path = os.path.join(meeting.get_materials_path(),doc.type.slug,doc.name) files = glob.glob(path + '.*') @@ -191,14 +190,14 @@ def process_pdfs(request, meeting_num): warn_count = 0 count = 0 meeting = get_object_or_404(Meeting, number=meeting_num) - ppt = Document.objects.filter(session__meeting=meeting,type='slides',external_url__endswith='.ppt').exclude(states__slug='deleted') - pptx = Document.objects.filter(session__meeting=meeting,type='slides',external_url__endswith='.pptx').exclude(states__slug='deleted') + ppt = Document.objects.filter(session__meeting=meeting,type='slides',uploaded_filename__endswith='.ppt').exclude(states__slug='deleted') + pptx = Document.objects.filter(session__meeting=meeting,type='slides',uploaded_filename__endswith='.pptx').exclude(states__slug='deleted') for doc in itertools.chain(ppt,pptx): - base,ext = os.path.splitext(doc.external_url) + base,ext = os.path.splitext(doc.uploaded_filename) pdf_file = base + '.pdf' path = os.path.join(settings.SECR_PROCEEDINGS_DIR,meeting_num,'slides',pdf_file) if os.path.exists(path): - doc.external_url = pdf_file + doc.uploaded_filename = pdf_file e = DocEvent.objects.create( type='changed_document', by=Person.objects.get(name="(System)"), @@ -313,8 +312,8 @@ def select(request, meeting_num): # count PowerPoint files waiting to be converted # TODO : This should look at SessionPresentation instead - ppt = Document.objects.filter(session__meeting=meeting,type='slides',external_url__endswith='.ppt').exclude(states__slug='deleted') - pptx = Document.objects.filter(session__meeting=meeting,type='slides',external_url__endswith='.pptx').exclude(states__slug='deleted') + ppt = Document.objects.filter(session__meeting=meeting,type='slides',uploaded_filename__endswith='.ppt').exclude(states__slug='deleted') + pptx = Document.objects.filter(session__meeting=meeting,type='slides',uploaded_filename__endswith='.pptx').exclude(states__slug='deleted') ppt_count = ppt.count() + pptx.count() return render(request, 'proceedings/select.html', { diff --git a/ietf/secr/utils/document.py b/ietf/secr/utils/document.py index b1a8e32ca..535758425 100644 --- a/ietf/secr/utils/document.py +++ b/ietf/secr/utils/document.py @@ -5,9 +5,9 @@ def get_full_path(doc): ''' import os - if doc.type_id not in ('slides','agenda','minutes') or not doc.external_url: + if doc.type_id not in ('slides','agenda','minutes') or not doc.uploaded_filename: return None - return os.path.join(doc.get_file_path(), doc.external_url) + return os.path.join(doc.get_file_path(), doc.uploaded_filename) def get_rfc_num(doc): qs = doc.docalias_set.filter(name__startswith='rfc') diff --git a/ietf/settings.py b/ietf/settings.py index cf03f042b..a47884eb1 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -611,8 +611,8 @@ DOC_HREFS = { "slides": "https://www.ietf.org/slides/{doc.name}-{doc.rev}", "conflrev": "https://www.ietf.org/cr/{doc.name}-{doc.rev}.txt", "statchg": "https://www.ietf.org/sc/{doc.name}-{doc.rev}.txt", - "liaison": "%s{doc.external_url}" % LIAISON_ATTACH_URL, - "liai-att": "%s{doc.external_url}" % LIAISON_ATTACH_URL, + "liaison": "%s{doc.uploaded_filename}" % LIAISON_ATTACH_URL, + "liai-att": "%s{doc.uploaded_filename}" % LIAISON_ATTACH_URL, } MEETING_DOC_HREFS = { @@ -620,7 +620,7 @@ MEETING_DOC_HREFS = { "minutes": "/meeting/{meeting.number}/materials/{doc.name}-{doc.rev}", "slides": "/meeting/{meeting.number}/materials/{doc.name}-{doc.rev}", "recording": "{doc.external_url}", - "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.external_url}", + "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.uploaded_filename}", } MEETING_DOC_OLD_HREFS = { @@ -628,7 +628,7 @@ MEETING_DOC_OLD_HREFS = { "minutes": "/meeting/{meeting.number}/materials/{doc.name}", "slides": "/meeting/{meeting.number}/materials/{doc.name}", "recording": "{doc.external_url}", - "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.external_url}", + "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.uploaded_filename}", } # For http references to documents without a version number (that is, to the current version at the time of reference) @@ -637,7 +637,7 @@ MEETING_DOC_GREFS = { "minutes": "/meeting/{meeting.number}/materials/{doc.name}", "slides": "/meeting/{meeting.number}/materials/{doc.name}", "recording": "{doc.external_url}", - "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.external_url}", + "bluesheets": "https://www.ietf.org/proceedings/{meeting.number}/bluesheets/{doc.uploaded_filename}", } # Override this in settings_local.py if needed