altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Third part of document read refactoring, after [14406] and [14410]. This replaces all usage of the non-unicode-aware get_document_content() function with unicode-aware Document.text() or Document.text_or_error() methods. This was triggered by yet another report of unicode content not being shown properly, and should fix all instances of document (drafts, agendas, minutes, etc.) display in the datatracker not handling unicode characters properly.

Browse source 
- Legacy-Id: 14411
Note: SVN reference [14406] has been migrated to Git commit 967ece7e7d

Note: SVN reference [14410] has been migrated to Git commit 660c81c272
This commit is contained in:
Henrik Levkowetz 2017-12-11 16:23:51 +00:00
parent 660c81c272
commit 1f976da5c1
9 changed files with 27 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
ietf/doc/mails.py
View file

@ -1,6 +1,5 @@
# generation of mails
import os
import textwrap, datetime
from django.template.loader import render_to_string
@ -13,11 +12,11 @@ import debug # pyflakes:ignore
from ietf.utils.mail import send_mail, send_mail_text
from ietf.ipr.utils import iprs_from_docs, related_docs
from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent
from ietf.doc.utils import needed_ballot_positions, get_document_content
from ietf.doc.utils import needed_ballot_positions
from ietf.group.models import Role
from ietf.doc.models import Document
from ietf.mailtrigger.utils import gather_address_lists
from ietf.utils import log
def email_state_changed(request, doc, text, mailtrigger_id=None):
(to,cc) = gather_address_lists(mailtrigger_id or 'doc_state_edited',doc=doc)
@ -512,18 +511,7 @@ def send_review_possibly_replaces_request(request, doc, submitter_info):
def email_charter_internal_review(request, charter):
addrs = gather_address_lists('charter_internal_review',doc=charter,group=charter.group)
filename = '%s-%s.txt' % (charter.canonical_name(),charter.rev)
charter_text = get_document_content(
filename,
os.path.join(settings.CHARTER_PATH,filename),
split=False,
markup=False,
)
utext = charter.text_or_error() # pyflakes:ignore
if charter_text and charter_text != utext and not 'Error; cannot read' in charter_text:
debug.show('charter_text[:64]')
debug.show('utext[:64]')
log.assertion('charter_text == utext')
charter_text = charter.text_or_error() # pyflakes:ignore
send_mail(request, addrs.to, settings.DEFAULT_FROM_EMAIL,
'Internal %s Review: %s (%s)'%(charter.group.type.name,charter.group.name,charter.group.acronym),

8
ietf/doc/models.py
View file

@ -437,6 +437,10 @@ class DocumentInfo(models.Model):
def text(self):
path = self.get_file_name()
root, ext = os.path.splitext(path)
txtpath = root+'.txt'
if ext != '.txt' and os.path.exists(txtpath):
path = txtpath
try:
with open(path, 'rb') as file:
raw = file.read()
@ -450,7 +454,7 @@ class DocumentInfo(models.Model):
return text
def text_or_error(self):
return self.text() or "Error; cannot read (%s)"%self.get_file_name()
return self.text() or "Error; cannot read '%s'"%self.get_base_name()
def htmlized(self):
name = self.get_base_name()
@ -607,7 +611,7 @@ class Document(DocumentInfo):
if not hasattr(self, '_canonical_name'):
name = self.name
if self.type_id == "draft" and self.get_state_slug() == "rfc":
a = self.docalias_set.filter(name__startswith="rfc").first()
a = self.docalias_set.filter(name__startswith="rfc").order_by('-name').first()
if a:
name = a.name
elif self.type_id == "charter":

11
ietf/doc/templatetags/ietf_filters.py
View file

@ -2,7 +2,6 @@
import re
import datetime
import os
import types
from email.utils import parseaddr
@ -16,9 +15,7 @@ from django.utils.html import strip_tags
import debug # pyflakes:ignore
from ietf.doc.models import ConsensusDocEvent
from ietf.doc.utils import get_document_content
from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped
from ietf.utils import log
register = template.Library()
@ -508,13 +505,7 @@ def emailwrap(email):
def document_content(doc):
if doc is None:
return None
path = os.path.join(doc.get_file_path(),doc.filename_with_rev())
content = get_document_content(doc.name,path,markup=False)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
return content
@register.filter

3
ietf/doc/utils.py
View file

@ -25,6 +25,7 @@ from ietf.ietfauth.utils import has_role
from ietf.utils import draft, text
from ietf.utils.mail import send_mail
from ietf.mailtrigger.utils import gather_address_lists
from ietf.utils import log
def save_document_in_history(doc):
"""Save a snapshot of document and related objects in the database."""
@ -299,7 +300,7 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):
return raw_content
def get_document_content(key, filename, split=True, markup=True):
#log.unreachable("2017-12-05")
log.unreachable("2017-12-05")
try:
with open(filename, 'rb') as f:
raw_content = f.read()

11
ietf/doc/views_conflict_review.py
View file

@ -12,7 +12,7 @@ import debug # pyflakes:ignore
from ietf.doc.models import ( BallotDocEvent, BallotPositionDocEvent, DocAlias, DocEvent,
Document, NewRevisionDocEvent, State )
from ietf.doc.utils import ( add_state_change_event, close_open_ballots,
create_ballot_if_not_open, get_document_content, update_telechat )
create_ballot_if_not_open, update_telechat )
from ietf.doc.mails import email_iana
from ietf.doc.forms import AdForm
from ietf.group.models import Role, Group
@ -253,14 +253,7 @@ def edit_ad(request, name):
def default_approval_text(review):
filename = "%s-%s.txt" % (review.canonical_name(), review.rev)
current_text = get_document_content(filename, os.path.join(settings.CONFLICT_REVIEW_PATH, filename), split=False, markup=False)
utext = review.text_or_error() # pyflakes:ignore
if current_text and current_text != utext and not 'Error; cannot read' in current_text:
debug.show('current_text[:64]')
debug.show('utext[:64]')
log.assertion('current_text == utext')
current_text = review.text_or_error() # pyflakes:ignore
conflictdoc = review.relateddocument_set.get(relationship__slug='conflrev').target.document
if conflictdoc.stream_id=='ise':
receiver = 'Independent Submissions Editor'

54
ietf/doc/views_doc.py
View file

@ -47,7 +47,7 @@ from ietf.doc.models import ( Document, DocAlias, DocHistory, DocEvent, BallotDo
ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent,
IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS )
from ietf.doc.utils import ( add_links_in_new_revision_events, augment_events_with_revision,
can_adopt_draft, get_chartering_type, get_document_content, get_tags_for_stream_id,
can_adopt_draft, get_chartering_type, get_tags_for_stream_id,
needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
get_initial_notify, make_notify_changed_event, make_rev_history, default_consensus,
add_events_message_info, get_unicode_document_content, build_doc_meta_block)
@ -66,7 +66,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
from ietf.review.models import ReviewRequest
from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_docs
from ietf.review.utils import no_review_from_teams_on_doc
from ietf.utils import markup_txt, log
from ietf.utils import markup_txt
from ietf.utils.text import maybe_split
@ -185,15 +185,7 @@ def document_main(request, name, rev=None):
if doc.get_state_slug() == "rfc":
# content
filename = name + ".txt"
content = get_document_content(filename, os.path.join(settings.RFC_PATH, filename),
split_content, markup=True)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
# file types
@ -221,15 +213,7 @@ def document_main(request, name, rev=None):
content = "This RFC is not available in plain text format."
split_content = False
else:
filename = "%s-%s.txt" % (draft_name, doc.rev)
content = get_document_content(filename, os.path.join(settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR, filename),
split_content, markup=True)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
# file types
@ -451,14 +435,7 @@ def document_main(request, name, rev=None):
))
if doc.type_id == "charter":
filename = "%s-%s.txt" % (doc.canonical_name(), doc.rev)
content = get_document_content(filename, os.path.join(settings.CHARTER_PATH, filename), split=False, markup=True)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(content)
ballot_summary = None
@ -502,12 +479,7 @@ def document_main(request, name, rev=None):
# This could move to a template
content = u"A conflict review response has not yet been proposed."
else:
content = get_document_content(filename, pathname, split=False, markup=True)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(content)
ballot_summary = None
@ -535,12 +507,7 @@ def document_main(request, name, rev=None):
# This could move to a template
content = u"Status change text has not yet been proposed."
else:
content = get_document_content(filename, pathname, split=False)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
ballot_summary = None
if doc.get_state_slug() in ("iesgeval"):
@ -593,12 +560,7 @@ def document_main(request, name, rev=None):
url = urlbase + extension
if extension == ".txt":
content = get_document_content(basename, pathname + extension, split=False)
utext = doc.text_or_error() # pyflakes:ignore
if content != utext:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error() # pyflakes:ignore
t = "plain text"
other_types.append((t, url))

10
ietf/doc/views_status_change.py
View file

@ -13,7 +13,7 @@ from ietf.doc.models import ( Document, DocAlias, State, DocEvent, BallotDocEven
BallotPositionDocEvent, NewRevisionDocEvent, WriteupDocEvent, STATUSCHANGE_RELATIONS )
from ietf.doc.forms import AdForm
from ietf.doc.lastcall import request_last_call
from ietf.doc.utils import get_document_content, add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open
from ietf.doc.utils import add_state_change_event, update_telechat, close_open_ballots, create_ballot_if_not_open
from ietf.doc.views_ballot import LastCallTextForm
from ietf.group.models import Group
from ietf.iesg.models import TelechatDate
@ -281,13 +281,7 @@ def newstatus(relateddoc):
def default_approval_text(status_change,relateddoc):
filename = "%s-%s.txt" % (status_change.canonical_name(), status_change.rev)
current_text = get_document_content(filename, os.path.join(settings.STATUS_CHANGE_PATH, filename), split=False, markup=False)
utext = status_change.text_or_error() # pyflakes:ignore
if current_text and current_text != utext and not 'Error; cannot read' in current_text:
debug.show('current_text[:64]')
debug.show('utext[:64]')
log.assertion('current_text == utext')
current_text = status_change.text_or_error() # pyflakes:ignore
if relateddoc.target.document.std_level.slug in ('std','ps','ds','bcp',):
action = "Protocol Action"

10
ietf/meeting/forms.py
View file

@ -9,7 +9,6 @@ from django.forms import BaseInlineFormSet
import debug # pyflakes:ignore
from ietf.doc.models import Document, DocAlias, State, NewRevisionDocEvent
from ietf.doc.utils import get_document_content
from ietf.group.models import Group
from ietf.ietfauth.utils import has_role
from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones
@ -18,7 +17,6 @@ from ietf.meeting.helpers import is_meeting_approved, get_next_agenda_name
from ietf.message.models import Message
from ietf.person.models import Person
from ietf.utils.fields import DatepickerDateField, DurationField
from ietf.utils import log
# need to insert empty option for use in ChoiceField
# countries.insert(0, ('', '-'*9 ))
@ -220,13 +218,7 @@ class InterimSessionModelForm(forms.ModelForm):
self.initial['time'] = self.instance.official_timeslotassignment().timeslot.time
if self.instance.agenda():
doc = self.instance.agenda()
path = os.path.join(doc.get_file_path(), doc.filename_with_rev())
content = get_document_content(os.path.basename(path), path, markup=False)
utext = doc.text_or_error() # pyflakes:ignore
if content and content != utext and not 'Error; cannot read' in content:
debug.show('content[:64]')
debug.show('utext[:64]')
log.assertion('content == utext')
content = doc.text_or_error()
self.initial['agenda'] = content

13
ietf/secr/telechat/views.py
View file

@ -1,4 +1,3 @@
import os
import datetime
from django.contrib import messages
@ -9,7 +8,7 @@ from django.utils.functional import curry
import debug # pyflakes:ignore
from ietf.doc.models import DocEvent, Document, BallotDocEvent, BallotPositionDocEvent, BallotType, WriteupDocEvent
from ietf.doc.utils import get_document_content, add_state_change_event
from ietf.doc.utils import add_state_change_event
from ietf.person.models import Person
from ietf.doc.lastcall import request_last_call
from ietf.doc.mails import email_state_changed
@ -17,8 +16,6 @@ from ietf.iesg.models import TelechatDate, TelechatAgendaItem, Telechat
from ietf.iesg.agenda import agenda_data, get_doc_section
from ietf.ietfauth.utils import role_required
from ietf.secr.telechat.forms import BallotForm, ChangeStateForm, DateSelectForm, TELECHAT_TAGS
from ietf.utils import log
'''
EXPECTED CHANGES:
@ -71,13 +68,7 @@ def get_doc_writeup(doc):
if latest:
writeup = latest.text
elif doc.type_id == 'conflrev':
path = os.path.join(doc.get_file_path(),doc.filename_with_rev())
writeup = get_document_content(doc.name,path,split=False,markup=False)
utext = doc.text_or_error() # pyflakes:ignore
if writeup and writeup != utext and not 'Error; cannot read' in writeup:
debug.show('writeup[:64]')
debug.show('utext[:64]')
log.assertion('writeup == utext')
writeup = doc.text_or_error() # pyflakes:ignore
return writeup
def get_last_telechat_date():