Merged in changes from trunk up to r17584.

- Legacy-Id: 17593

.gitignore

@@ -26,6 +26,7 @@
 /attic
 /bin
 /etc
+/env
 /ghostdriver.log
 /htmlcov
 /include
@@ -48,3 +49,5 @@
 /trunk37
 /unix.tag
 /tmp-nomcom-public-keys-dir
+*.pyc
+__pycache__

bin/mergeready

@@ -87,16 +87,17 @@ for opt, value in opts:
     if   opt in ["-h", "--help"]: # Output this help, then exit
         print( __doc__ % locals() )
         sys.exit(1)
-    elif opt in ["-v", "--version"]: # Output version information, then exit
+    elif opt in ["-V", "--version"]: # Output version information, then exit
         print( program, version )
         sys.exit(0)
-    elif opt in ["-V", "--verbose"]: # Output version information, then exit
+    elif opt in ["-v", "--verbose"]: # Output version information, then exit
         opt_verbose += 1
 
 # ----------------------------------------------------------------------
 def say(s):
     sys.stderr.write("%s\n" % (s))
 
+
 # ----------------------------------------------------------------------
 def note(s):
     if opt_verbose:
@@ -204,6 +205,7 @@ for line in pipe('svn propget svn:mergeinfo .').splitlines():
                 write_cache = True
         mergeinfo[line] = merged
     merged_revs.update(merged)
+note('')
 
 if write_cache:
     cache[repo] = mergeinfo
@@ -227,7 +229,7 @@ def get_changeset_list_from_file(repo, filename):
             if line.startswith('#') or line == "":
                 continue
             try:
-                note("   '%s'" % line)
+                #note("   '%s'" % line)
                 parts = line.split()
                 if len(parts) >1 and parts[1] == '@':
                     branch, rev = parts[0], parts[2]
@@ -253,7 +255,7 @@ def get_changeset_list_from_file(repo, filename):
 def get_ready_commits(repo, tree):
     list = []
     note("Getting ready commits from '%s'" % tree)
-    cmd = 'svn log -v -r %s:HEAD %s/%s/' % ((head-500), repo, tree)
+    cmd = 'svn log -v -r %s:HEAD %s/%s/' % ((head-200), repo, tree)
     if opt_verbose > 1:
         note("Running '%s' ..." % cmd)
     commit_log = pipe(cmd)
@@ -273,7 +275,7 @@ def get_ready_commits(repo, tree):
                 note("   %s  %s: %s@%s" % (when.strftime("%Y-%m-%d %H:%MZ"), who, branch, rev))
                 list += [(rev, repo, branch),]
             elif rev in merged_revs and not branch == merged_revs[rev]:
-                sys.stderr.write('Rev %s: %s != %s' % (rev, branch, merged_revs[rev]))                
+                sys.stderr.write('Rev %s: %s != %s\n' % (rev, branch, merged_revs[rev]))                
             else:
                 pass
         else:
@@ -286,10 +288,8 @@ ready += get_changeset_list_from_file(repo, '../ready-for-merge')
 hold   = get_changeset_list_from_file(repo, 'hold-for-merge')
 hold  += get_changeset_list_from_file(repo, '../hold-for-merge')
 ready += get_ready_commits(repo, 'personal')
-ready += get_ready_commits(repo, 'branch/amsl')
 ready += get_ready_commits(repo, 'branch/iola')
 ready += get_ready_commits(repo, 'branch/dash')
-ready += get_ready_commits(repo, 'branch/proceedings')
 
 ready_commits = {}
 all_commits = {}
@@ -328,7 +328,9 @@ for entry in ready:
         else:
             raise
     # 
-    merge_path = os.path.join(*path.split(os.path.sep)[:4])
+    dirs = path.split(os.path.sep)
+    dirs = dirs[:dirs.index('ietf')] if 'ietf' in dirs else dirs[:4]
+    merge_path = os.path.join(*dirs)
     if not (rev, repo, merge_path) in hold:
         output_line = "%s  %-24s ^/%s@%s" % (when.strftime("%Y-%m-%d_%H:%MZ"), who+":", merge_path, rev)
         all_commits[when] = (rev, repo, branch, who, merge_path)
@@ -388,10 +390,11 @@ for key in keys:
 keys = list(not_passed.keys())
 keys.sort()
 if len(keys) > 0:
-    sys.stderr.write("Commits marked ready which haven't passed the test suite:\n")
+    print("")
+    print("Commits marked ready which haven't passed the test suite:\n")
     for key in keys:
-        sys.stderr.write(not_passed[key]+'\n')
+        print(not_passed[key])
-    sys.stderr.write('\n')
+    print('')
 
 keys = list(ready_commits.keys())
 keys.sort()

bin/mkdevbranch

@@ -78,12 +78,13 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
 # Option parsing
 
 # Options
-shortopts=hm:M:vV
+shortopts=hm:M:nvV
-longopts=help,meeting=,message=,verbose,version
+longopts=help,meeting=,message=,dry-run,verbose,version
 
 # Default values
 num=""
 msg=""
+do=""
 
 if   [ "$(uname)" = "Linux" ]; then
     args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
@@ -103,6 +104,7 @@ while true ; do
 	-h| --help)	usage; exit;;		# Show this help, then exit
 	-m| --meeting)	num=$2; shift;;		# Specify the IETF meeting number
 	-M| --message)	msg=$2; shift;;		# Specify extra message text
+	-n| --dry-run)  do="echo -- ==>";;	# Only show what would be done
 	-v| --verbose)	VERBOSE=1;;		# Be more talkative
 	-V| --version)	version; exit;;		# Show program version, then exit
 	--)		shift; break;;
@@ -128,8 +130,8 @@ function mksvndir() {
     who=$1
     if [ "$2" ]; then dir=$2; else dir=$who; fi
     if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir >/dev/null 2>&1 ; then
-	echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
+	$do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
-	svn mkdir https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code"
+	$do svn mkdir https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code"
     else
 	echo "Repository area personal/$dir is already in place."
     fi
@@ -143,7 +145,7 @@ cd $progdir
 
 if [ "$who" ]; then
     mksvndir $who
-    svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target"
+    $do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target"
     echo "New branch: ^/personal/$who/$target"
 else
     [ "$msg" ] && msg="
@@ -154,11 +156,21 @@ $msg
 	trac-admin /www/tools.ietf.org/tools/ietfdb wiki export IETF${n}SprintSignUp	\
 	    | egrep "^\|\|" | tail -n +2 | python -c '
 import sys, re
-afile = open("aliases")
+with open("aliases") as afile:
-aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() ])
+    try:
+	aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
+    except ValueError:
+	sys.stderr.write([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
+	raise
 
 for line in sys.stdin:
-    blank, name, email, rest = line.strip().split("||", 3)
+    try:
+	blank, name, email, rest = line.strip().split("||", 3)
+	email = email.strip()
+    except ValueError:
+	sys.stderr.write(line+"\n")
+	raise
+	
     login, dummy  = re.split("[@.]", email, 1)
     if email in aliases:
 	login = aliases[email]
@@ -171,9 +183,9 @@ for line in sys.stdin:
 	echo "$login ($name <$email>):"
 	mksvndir $login
 	if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target >/dev/null 2>&1 ; then
-	    echo "  creating $target branch for $login ($name)."
+	    $do echo "  creating $target branch for $login ($name)."
-	    svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name"	\
+	    $do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name"	\
-	    && mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b henrik@levkowetz.com <<-EOF
+	    && $do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b henrik@levkowetz.com <<-EOF
 		Hi,
                 $msg
 		This mail has been automatically generated by the $program script.
@@ -199,7 +211,7 @@ for line in sys.stdin:
 
 	EOF
 	else
-	    echo "  branch personal/$login/$target already exists."
+	    $do echo "  branch personal/$login/$target already exists."
 	fi
     done
 fi

bin/mkpatch

@@ -71,8 +71,8 @@ trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)";
 # Option parsing
 
 # Options
-shortopts=c:r:hvV
+shortopts=c:or:hvV
-longopts=change=,revision=,help,verbose,version
+longopts=change=,overwrite,revision=,help,verbose,version
 
 # Default values
 
@@ -92,7 +92,7 @@ fi
 while true ; do
     case "$1" in
 	-c| --change)	CHG="$2"; shift;;	# the change made by revision ARG
-	-r| --revision)	REV="$2"; shift;;	# the change made between revisions REV
+        -o| --overwrite) OVER=1;;                # overwrite any existing patch file
 	-h| --help)	usage; exit;;		# Show this help, then exit
 	-v| --verbose)	VERBOSE=1;;	# Be more talkative
 	-V| --version)	version; exit;;		# Show program version, then exit
@@ -105,16 +105,19 @@ done
 # ----------------------------------------------------------------------
 # The program itself
 
-if [ $# -lt 2 ]; then die "Expected patch name and file list on the command line."; fi
+if   [ "$CHG" ]; then
-if [[ $1 =~ / ]]; then die "Expected a patch name, but the first argument to $program seems to be a file path: '$1'"; fi
+  name=$(echo $(svn log -c $CHG | sed -r -e '/^---/d' -e '/^r[0-9]+/d' -e '/^$/d' -e 's/Merged in \[[0-9]+\] from [^:]+..//' ) | sed -r -e 's/(.*)/\L\1/' -e 's/[^[:alnum:]]/-/g' -e 's/-+/-/g' | cut -c 1-40)
-
+  name="$name-c$CHG"
-name=$1; shift;
+else
-if [ "$CHG" ]; then name="$name-c$CHG"; fi
+  if [ $# -lt 2 ]; then die "Expected patch name and file list on the command line."; fi
+  if [[ $1 =~ / ]]; then die "Expected a patch name, but the first argument to $program seems to be a file path: '$1'"; fi
+  name=$1; shift;
+fi
 
 patchfile=$progdir/../../patches/$(date +%Y-%m-%d)-$name.patch
-if [ -e $patchfile ]; then die "Patchfile $patchfile already exists"; fi
+if [ -e $patchfile -a ! -n "$OVER" ]; then die "Patchfile $patchfile already exists"; fi
 svn diff ${CHG:+ -c $CHG} ${REV:+ -r $REV} "$@" > $patchfile
 less $patchfile
 echo ""
 echo ""
-echo "Patch is in $patchfile."
+echo "Patch is in $patchfile"

bin/mkrelease

@@ -66,6 +66,10 @@ function note() {
 	if [ -n "$VERBOSE" ]; then echo -e "\n$*"; fi
 }
 
+function check() {
+  [ "$(which $1)" ] || die "could not find the '$1' command.  $2"
+}
+
 # ----------------------------------------------------------------------
 function version() {
 	echo -e "$program $version"
@@ -119,6 +123,11 @@ while true ; do
     shift
 done
 
+# ----------------------------------------------------------------------
+# Check some requirements
+
+check bower "It is required to update web resources.  Install with npm."
+
 # ----------------------------------------------------------------------
 # The program itself
 
@@ -212,6 +221,7 @@ if [ -z "$IGNORE_RESOURCES" ]; then
     $do svn commit ietf/externals/static -m "Updated bower-managed static web assets"
     # Get rid of bower-installed files which we don't use:
     $do rm -rf ietf/externals/static/datatracker/
+    $do rm -rf ietf/externals/static/jquery.cookie/
     $do rm -f $(svn st ietf/externals/ | grep '^\?' | awk '{print $2}')
 fi
 
@@ -244,7 +254,9 @@ if [ -d ../coverage ]; then
     rsync -a static/coverage/ ../coverage/$VER/
 fi
 
-contributors=$(echo "$changes" | sed 's/\.[ \t\n]/ /'| tr -c "a-z0-9.@-" "\n" | sort | uniq | grep '@' | sed -r -e 's/^\.+//' -e 's/\.+$//' -e 's/^/-c /' || true)
+contributors=$(echo "$changes" | gawk '/^  \* Merged in \[[0-9]+\] from [^: ]+/ {sub(":",""); print $6;}' | sort | uniq)
+note "Contributors:
+$contributors"
 
 note "Setting the current time on the release notes in the changelog file ..."
 $do sed -r -i -e "1,/^ -- /s/([A-Za-z-]+ <[a-z0-9.-]+@[a-z0-9.-]+>  ).*$/\1$(TZ=UTC date +'%d %b %Y %H:%M:%S %z')/" changelog

changelog

@@ -1,3 +1,323 @@
+ietfdb (6.124.0) ietf; urgency=medium
+
+  **Enhanced 'Upcoming Meetings' page, and more**
+
+  * Added links to agenda/materials pop-up, materials download, etherpad, 
+    jabber room, and webex call-in session for interims on the upcoming 
+    meetings page.  With the earlier changes from [17555], this fixes issue 
+    #2937.
+
+  * Changed the IPR patent number regex to permit space between country 
+    code and serial number, and expanded on the help text for the IPR patent 
+    number field.
+
+  * Added verification of response data to IphoneAppJsonTests
+
+  * Merged in [17564] from pusateri@bangj.com:
+    Added interim meetings to agenda.json API. Fixes #2946.
+
+  * Merged in [17562] from jennifer@painless-security.com:
+    Add tooltips with doc name to 'updates' and 'obsoletes' links. Fixes 
+    #2866;  
+
+  * Fixed a Py2/3 issue in the djangobwr's bower_install command
+
+  * Added a check for availability of 'bower' in bin/mkrelease.
+
+  * Cleaned up the contributors list in bin/mkrelease a bit.
+
+  * Merged in [17557] from fenton@bluepopcorn.net:
+    Provide more consistent links to people pages. Fixes #2918.  
+
+  * Changed an obsolete document.href() to document.get_href().  Fixes 
+    issue #2945.
+
+  * Tweaked the upcoming calendar and calendar entries slightly, to render 
+    with times first and on two lines on narrow screens.
+
+  * Merged in [17555] from rjsparks@nostrum.com:
+    Remove the not-quite-working customization widgets from 
+    /meeting/upcoming and /meeting/past. Simplify those views. Correct the list 
+    of sessions on those pages when one interim has more than one session. 
+    Fixes #2938. Partially addresses #2937.  
+
+  * Prevent an exception on missing author.email.person when listing author 
+    emails.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  03 Apr 2020 10:04:43 +0000
+
+
+ietfdb (6.123.1) ietf; urgency=medium
+
+  **Fixes for meeting-related issues**
+
+  * Merged in [17543] from rjsparks@nostrum.com:
+    Repaired construction of group_hierarchy used for the customisation 
+    controls at /meeting/upcoming. Fixes #2940.  
+
+  * Merged in [17542] from rjsparks@nostrum.com:
+    Show calendar entries on /meeting/upcoming in utc. Show end times. 
+    Partially addresses #2936.  
+
+  * Don't show agenda buttons for Meetecho recordings (after the session 
+    concludes) if there isn't a Meetecho UrlResource.  Fixes issue #2934
+
+  * Merged in [17538] from rjsparks@nostrum.com:
+    Allow an out-of-area AD assigned as the AD for a WG to approve interim 
+    requests for that WG. Fixes #2930.  
+
+  * Changed the object factory instances of nomcom private key and cert to 
+    be byte objects (matching the production settings), and fixed the issue 
+    with nomcom key handling under Py3 found by fenton@bluepopcorn.net.  Did 
+    some renaming in nomcom/tests.py to better match setup/teardown function 
+    names to functionality.
+
+  * Merged in [17521] from housley@vigilsec.com:
+    Improve performance of log.assertion() and log.unreachable() 
+
+  * Merged in [17505] from housley@vigilsec.com:
+    Improve performance of many document list pages 
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  27 Mar 2020 14:27:24 +0000
+
+
+ietfdb (6.123.0) ietf; urgency=medium
+
+  **IETF 107 code sprint**
+
+  This release contains datatracker bug fixes and enhancements from the
+  IETF-107 Code Sprint, our first virtual code sprint.  It was a different
+  experience, and enjoyable despite not getting to sit down with a beer
+  together afterwards.  A lot of good contributions were made; Thanks to
+  everyone who contributed!
+
+  * Merged in [17496] from rjsparks@nostrum.com:
+    Remove the rest of the log.assertions checking that iesg_state existed in
+    places we expected it to.  Removed unnecessary imports.
+
+  * Changed the page for upcoming meetings to show the current IETF meeting 
+    for 7 days from its start date, while interims are shown for today and 
+    forward.  Also changed the upcoming.ics calendar to show future sessions, 
+    even if the meeting to which they belong started in the past.  This 
+    improves on [17518].
+
+  * Changed the starting point of display of upcoming meetings to be 7 days 
+    before today, rather than today, to let meetings linger a bit in the 
+    listing and iCalendar file after the meeting has started.  Triggered by an 
+    observation from resnick@episteme.net about IETF 107 sessions disappearing 
+    from 'upcoming.ics' on meeting week Monday.
+
+  * Merged in [17495] from rjsparks@nostrum.com:
+    Removes a log.assertion() that was checking that we covered the edges when
+    we changed documents to always have an iesg state.  
+
+  * Merged in [17494] from rjsparks@nostrum.com:
+    Use current email addresses when we have them when listing document
+    authors. Fixes #1902.  
+
+  * Merged in [17493] from mahoney@nostrum.com:
+    Changed awkward IESG/IAB Nominating Committee names to just NomCom,
+    updated a ref. Fixes #2860.  
+
+  * Merged in [17492] from rcross@amsl.com:
+    On session request form, made the Special Requests field smaller and
+    display 200 character limit. Fixes #2875.  
+
+  * Merged in [17491] from rcross@amsl.com:
+    Prevent use of capital letters in group acronym. Fixes #2709.  
+
+  * Merged in [17490] from rjsparks@nostrum.com:
+    Basic regex validation on community rule entry form. Fixes #2928.  
+
+  * Merged in [17489] from rcross@amsl.com:
+    Removed redundant URL secr/groups/search because search page is 
+    available here secr/groups. Resolves issue with Add link. Fixes #2708.  
+
+  * Merged in [17488] from rcross@amsl.com:
+    Removed the drafts secretariat tool because this functionality is now 
+    provided by the core Datatracker. Moved ID reports to proceedings tool. 
+    Fixes #1655.  
+
+  * Merged in [17487] from rjsparks@nostrum.com:
+    Let chairs know what to do after material submission uploads have been 
+    cut off. Fixes #2887.  
+
+  * Merged in [17486] from valery@smyslov.net:
+    Added docker/run modifications to support Cygwin.
+
+  * Merged in [17484] from valery@smyslov.net:
+    When requesting a new WG session, and retrieving information about the
+    previous session, look back to the previous time the group met, instead of
+    simply checking the previous IETF meeting and maybe not finding any
+    information to retrieve.
+
+  * Merged in [17483] from peter@akayla.com:
+    Changed things so that only WGs/RGs can be closed, per RJS.  Fixes #1578.   
+
+  * Merged in [17466] from rcross@amsl.com:
+    Added a migration to cancel 107 sessions 
+
+  * Added a check to see if any files matching the submitted draft name and
+    revision already exists on disk in the active drafts or archived drafts
+    directories, and if so reject the submission.  Fixes issue #2908
+
+  * Made sure to strip possible mail header field values of whitespace 
+    before applying email.utils.unquite().  Resolution by kivinen@iki.fi,  
+    Fixes issue #2899.
+
+  * Merged in [17480] from rjsparks@nostrum.com:
+    Show UTC times in interim announcements if the interim has a non-UTC 
+    timzone. Fixes #2922.  
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  24 Mar 2020 17:53:45 +0000
+
+
+ietfdb (6.122.0) ietf; urgency=medium
+
+  **Added agenda webex URL support, and meeting-related tweaks and bugfixes**
+
+  * Added webex URL to agenda.ics if Room.webex_url is non-empty.  Fixes issue
+    #2926.
+
+  * Added another check to the check_draft_event_revision_integrity management
+    command, and refined it somewhat.
+
+  * Added a utility function to convert objects to dictionaries (for
+    comparisons, for instance).
+
+  * Added a --dry-run option to bin/mkdevbranch, and added some exception
+    handling.
+
+  * Help tablesorter see 'Month Year' dates as dates with a hidden day digit.
+    Fixes issue #2921.
+
+  * Refactored and extended check_draft_event_revision_integrity a bit.
+
+  * Tweaked bin/mkpatch some for -c handling
+
+  * Merged in [17442] from rjsparks@nostrum.com: Allow area groups to request
+    interim meetings. Fixed #2919.  
+
+  * Additional tweaks to bin/mkpatch; removing buggy -r option.
+
+  * Added automatic naming to bin/mkpatch when changeset or revision range is
+    given.
+
+  * Added WebEx room resource name, query method and template logic to show
+    WebEx room resources.
+
+  * Removed a debug statement
+
+  * Made links from agenda room names to floorplans conditional on the room
+    having a floor plan set.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  20 Mar 2020 19:50:12 +0000
+
+
+ietfdb (6.121.0) ietf; urgency=medium
+
+  **Tweaks for wholly virtual meeting, for IETF-107**
+
+  * Added code to show a webex call-in button on the agenda page if the session
+    agenda-note contains and IETF webex URL.
+
+  * Merged in [17425] from rjsparks@nostrum.com: Make required AD approval of
+    virtual interims configurable. Fixes #2912.  
+
+  * Added a management command to check draft event revision numbers.  To be
+    extended for other checks.
+
+  * Merged in [17419] from rjsparks@nostrum.com: Don't warn about idcutoff
+    when the cutoff is after the meeting starts.  Fixes #2907.  
+
+  * Merged in [17418] from rjsparks@nostrum.com: Correctly represent cancelled
+    sessions in ics files. Fixes #2905.  
+
+  * Merged in [17396] from rjsparks@nostrum.com: Move charters for replaced
+    groups to a new replaced state. Close any outstanding ballots on them.
+    Fixes #2889, #2873, and #1286.  
+
+  * Avoid trying to open meeting documents with empty .uploaded_filename.
+
+  * Added a progress bar for verbosity=1 of the community list index update
+    command.
+
+  * Merged back fixes from production
+
+  * Corrected the extent of a try/except block, moving more code inside the
+    block.  Fixes a submission exception that should just be a document error
+    reported back to the user.
+
+  * Added a guard against accessing attributes of None.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  13 Mar 2020 14:48:11 +0000
+
+
+ietfdb (6.120.0) ietf; urgency=medium
+
+  **Submission API changes, Py2/3 transition fixes**
+
+  * Added the ability to use the submission API with active secondary account
+    email addresses.  Fixes issue #2639.
+
+  * Tweaked the ReviewAssignmentAdmin, adding a raw_id_field.
+
+  * Replaced most cases of using of urlopen(), instead using the higher-level
+    'requests' module where it simplifies the code.
+
+  * Added a data migration to fix up incorrect external URLs to mailarchive.
+
+  * Fixed a Py2/3 issue with review.mailarchive.construct_query_url().
+
+  * Renamed a migration to conform to migration naming conventions, using
+    underscores instead of dashes in the name.
+
+  * Py2/3 compatibility tweaks for pyflakes.
+
+  * Changed some cases of urlopen() to use requests.get()
+
+  * Python3 is more ticklish about comparing strings to None than Py2.  Fixed
+    an issue with this in generate_sort_key() for document searches.
+
+  * Fixed a Py2/3 issue in the pyflakes management command, and tweaked the
+    verbose output format.
+
+  * Merged back production changes to two scripts indirectly called by
+    /a/www/www6s/scripts/run-ietf-report, through
+    /a/www/www6s/scripts/run-report.
+
+  * Changed the release script to not pick up other email addresses than those
+    of contributors from the release notes.
+
+  * Tweaked the check_referential_integrity management command verbose output.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  07 Mar 2020 22:55:58 +0000
+
+
+ietfdb (6.119.1) ietf; urgency=medium
+
+  **Py2/3 fixes, Change to use the "requests" lib instead of urlopen()**
+
+  * Py2/3 compatibility tweaks for pyflakes.
+
+  * Changed some cases of urlopen() to use requests.get()
+
+  * Python3 is more ticklish about comparing strings to None than Py2.  
+    Fixed an issue with this in generate_sort_key() for document searches.
+
+  * Fixed a Py2/3 issue in the pyflakes management command, and tweaked the 
+    verbose output format.
+
+  * Merged back production changes to two scripts indirectly called by 
+    /a/www/www6s/scripts/run-ietf-report, through 
+    /a/www/www6s/scripts/run-report.
+
+  * Changed the release script to not pick up other email addresses than 
+    those of contributors from the release notes.
+
+ -- Henrik Levkowetz <henrik@levkowetz.com>  03 Mar 2020 11:23:46 +0000
+
+
 ietfdb (6.119.0) ietf; urgency=medium
 
   **Improved email handling, and roundup of Py2/3 conversion issues**

djangobwr/management/commands/bower_install.py

@@ -158,9 +158,9 @@ class Command(BaseCommand):
 
                             # Check if we need to copy the file at all.
                             if os.path.exists(dst_path):
-                                with open(src_path) as src:
+                                with open(src_path, 'br') as src:
                                     src_hash = hashlib.sha1(src.read()).hexdigest()
-                                with open(dst_path) as dst:
+                                with open(dst_path, 'br') as dst:
                                     dst_hash = hashlib.sha1(dst.read()).hexdigest()
                                 if src_hash == dst_hash:
                                     #print('{0} = {1}'.format(src_path, dst_path))

docker/run

@@ -133,6 +133,12 @@ if   [ "$(uname)" = "Darwin" ]; then
     CMD="open -a"
 elif [ "$(uname)" = "Linux" ]; then
     echo "Running on Linux."
+elif [[ $(uname) =~ CYGWIN.* ]]; then
+    echo "Running under Cygwin."
+    APP="Don't know how to start Docker when running under Cygwin"
+    CMD="echo"
+    MYSQLDIR=$(echo $MYSQLDIR | sed -e 's/^\/cygdrive\/\(.\)/\1:/')
+    WHO=$(echo $WHO | sed -e 's/^.*\\//' | tr -d \\r)
 else
     die "This script does not have support for your architecture ($(uname)); sorry :-("
 fi
@@ -184,7 +190,6 @@ else
     fi
 fi
 
-
 image=$(docker ps | grep "$REPO:$TAG" | awk '{ print $1 }')
 if [ "$image" ]; then
     if [ "$*" ]; then

hold-for-merge

@@ -1,5 +1,6 @@
 # -*- conf-mode -*-
 
+^/personal/mahoney/6.121.1.dev0@17473   # Test commit
 /personal/kivinen/6.94.2.dev0@16091	# Replaced by later commit
 /personal/rjs/6.104.1.dev0@16809	# Local changes, not for merge
 /personal/rjs/6.103.1.dev0@16761	# Fixed in a different manner in [16757]

ietf/__init__.py

@@ -5,13 +5,13 @@
 from . import checks                           # pyflakes:ignore
 
 # Don't add patch number here:
-__version__ = "6.119.1.dev0"
+__version__ = "6.124.1.dev0"
 
 # set this to ".p1", ".p2", etc. after patching
 __patch__   = ""
 
 __date__    = "$Date$"
 
-__rev__     = "$Rev$ (dev) Latest release: Rev. 17365 "
+__rev__     = "$Rev$ (dev) Latest release: Rev. 17582 "
 
 __id__      = "$Id$"

ietf/bin/find-submission-confirmation-email-in-postfix-log

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import io
 import os
 import sys
 

ietf/bin/generate-draft-aliases

@@ -28,7 +28,7 @@ TODO:
 """
 
 # boilerplate (from various other ietf/bin scripts)
-import os, sys, re
+import io, os, sys, re
 
 filename = os.path.abspath(__file__)
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))

ietf/bin/generate-wg-aliases

@@ -17,7 +17,7 @@ mail lists: -ads, and -chairs
 """
 
 # boilerplate (from various other ietf/bin scripts)
-import os, sys
+import io, os, sys
 
 filename = os.path.abspath(__file__)
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))

ietf/bin/iana-protocols-updates

@@ -3,9 +3,10 @@
 # This script requires that the proper virtual python environment has been
 # invoked before start
 
-import os
-import sys
 import datetime
+import os
+import requests
+import sys
 import syslog
 
 # boilerplate
@@ -19,7 +20,7 @@ import django
 django.setup()
 
 from django.conf import settings
-from ietf.sync.iana import fetch_protocol_page, parse_protocol_page, update_rfc_log_from_protocol_page
+from ietf.sync.iana import parse_protocol_page, update_rfc_log_from_protocol_page
 
 def chunks(l, n):
     """Split list l up in chunks of max size n."""
@@ -30,7 +31,7 @@ syslog.syslog("Updating history log with new RFC entries from IANA protocols pag
 # FIXME: this needs to be the date where this tool is first deployed
 rfc_must_published_later_than = datetime.datetime(2012, 11, 26, 0, 0, 0)
 
-text = fetch_protocol_page(settings.IANA_SYNC_PROTOCOLS_URL)
+text = requests.get(settings.IANA_SYNC_PROTOCOLS_URL).text
 rfc_numbers = parse_protocol_page(text)
 for chunk in chunks(rfc_numbers, 100):
     updated = update_rfc_log_from_protocol_page(chunk, rfc_must_published_later_than)

ietf/bin/report_id_activity

@@ -3,21 +3,20 @@
 # -*- Python -*-
 #
 
+# This script requires that the proper virtual python environment has been
+# invoked before start
+
 # Set PYTHONPATH and load environment variables for standalone script -----------------
 import os, sys
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 import django
 django.setup()
 # -------------------------------------------------------------------------------------
 
-from ietf.secr.drafts.reports import report_id_activity
+from ietf.secr.proceedings.reports import report_id_activity
 
-print report_id_activity(sys.argv[1], sys.argv[2]),
+print(report_id_activity(sys.argv[1], sys.argv[2]), end='')
 

ietf/bin/report_progress_report

@@ -3,24 +3,22 @@
 # -*- Python -*-
 #
 
+# This script requires that the proper virtual python environment has been
+# invoked before start
+
 # Set PYTHONPATH and load environment variables for standalone script -----------------
 import os, sys
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
 os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
 
-virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
-if os.path.exists(virtualenv_activation):
-    execfile(virtualenv_activation, dict(__file__=virtualenv_activation))
-
 import django
 django.setup()
 # -------------------------------------------------------------------------------------
 
-from ietf.secr.drafts.reports import report_progress_report
+from ietf.secr.proceedings.reports import report_progress_report
 
 # handle unicode characters before attempting to print
 output = report_progress_report(sys.argv[1], sys.argv[2])
-output = output.replace(unichr(160),' ')     # replace NO-BREAK SPACE with space
+output = output.replace(chr(160),' ')     # replace NO-BREAK SPACE with space
-output = output.encode('ascii','replace')
+print(output, end='')
-print output,

ietf/bin/rfc-editor-index-updates

@@ -4,15 +4,15 @@
 # invoked before start
 
 import datetime
+import io
 import json
 import os
+import requests
 import socket
 import sys
 import syslog
 import traceback
 
-from urllib.request import urlopen
-
 # boilerplate
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
 sys.path = [ basedir ] + sys.path
@@ -49,11 +49,10 @@ log("Updating document metadata from RFC index from %s" % settings.RFC_EDITOR_IN
 
 
 socket.setdefaulttimeout(30)
-rfc_index_xml = urlopen(settings.RFC_EDITOR_INDEX_URL)
+rfc_index_xml = requests.get(settings.RFC_EDITOR_INDEX_URL).text
-index_data = ietf.sync.rfceditor.parse_index(rfc_index_xml)
+index_data = ietf.sync.rfceditor.parse_index(io.StringIO(rfc_index_xml))
 
-rfc_errata_json = urlopen(settings.RFC_EDITOR_ERRATA_JSON_URL)
+errata_data = requests.get(settings.RFC_EDITOR_ERRATA_JSON_URL).json()
-errata_data = json.load(rfc_errata_json)
 
 if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS:
     log("Not enough index entries, only %s" % len(index_data))

ietf/bin/rfc-editor-queue-updates

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
+import io
 import os
+import requests
 import socket
 import sys
-from urllib.request import urlopen
 
 # boilerplate
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
@@ -21,8 +22,8 @@ from ietf.utils.log import log
 log("Updating RFC Editor queue states from %s" % settings.RFC_EDITOR_QUEUE_URL)
 
 socket.setdefaulttimeout(30)
-response = urlopen(settings.RFC_EDITOR_QUEUE_URL)
+response = requests.get(settings.RFC_EDITOR_QUEUE_URL).text
-drafts, warnings = parse_queue(response)
+drafts, warnings = parse_queue(io.StringIO(response))
 for w in warnings:
     log(u"Warning: %s" % w)
 

ietf/community/forms.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 
 
+import re
+
 from django import forms
 from django.db.models import Q
 
@@ -92,7 +94,12 @@ class SearchRuleForm(forms.ModelForm):
             f.required = True
 
     def clean_text(self):
-        return self.cleaned_data["text"].strip().lower() # names are always lower case
+        candidate_text = self.cleaned_data["text"].strip().lower() # names are always lower case
+        try:
+            re.compile(candidate_text)
+        except re.error as e:
+            raise forms.ValidationError(str(e))
+        return candidate_text
 
 
 class SubscriptionForm(forms.ModelForm):

ietf/doc/expire.py

@@ -25,7 +25,6 @@ def expirable_draft(draft):
     two functions need to be kept in sync."""
     if draft.type_id != 'draft':
         return False
-    log.assertion('draft.get_state_slug("draft-iesg")')
     return bool(expirable_drafts(Document.objects.filter(pk=draft.pk)))
 
 nonexpirable_states = []                # type: List[State]

ietf/doc/migrations/0030_fix_bytes_mailarch_url.py

@@ -0,0 +1,37 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-21 14:27
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import re
+
+from django.conf import settings
+from django.db import migrations
+
+
+def forward(apps, schema_editor):
+
+    Document                     = apps.get_model('doc', 'Document')
+
+    print('')
+    for d in Document.objects.filter(external_url__contains="/b'"):
+        match = re.search("^(%s/arch/msg/[^/]+/)b'([^']+)'$" % settings.MAILING_LIST_ARCHIVE_URL, d.external_url)
+        if match:
+            d.external_url = "%s%s" % (match.group(1), match.group(2))
+            d.save()
+            print('Fixed url #%s: %s' % (d.id, d.external_url))
+
+def reverse(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0029_add_ipr_event_types'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/doc/migrations/0031_set_state_for_charters_of_replaced_groups.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2020, All Rights Reserved
+# Generated by Django 1.11.28 on 2020-03-03 13:54
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+
+    Person = apps.get_model('person', 'Person')
+
+    Document = apps.get_model('doc','Document')
+    State = apps.get_model('doc','State')
+    BallotDocEvent = apps.get_model('doc','BallotDocEvent')
+
+    replaced_state = State.objects.create(type_id='charter', slug='replaced', name='Replaced', used=True, desc="This charter's group was replaced.", order = 0)
+    by = Person.objects.get(name='(System)')
+
+    for doc in Document.objects.filter(type_id='charter',states__type_id='charter',states__slug__in=['intrev','extrev'],group__state='replaced'):
+        doc.states.remove(*list(doc.states.filter(type_id='charter')))
+        doc.states.add(replaced_state)
+        ballot = BallotDocEvent.objects.filter(doc=doc, type__in=('created_ballot', 'closed_ballot')).order_by('-time', '-id').first()
+        if ballot and ballot.type == 'created_ballot':
+            e = BallotDocEvent(type="closed_ballot", doc=doc, rev=doc.rev, by=by)
+            e.ballot_type = ballot.ballot_type
+            e.desc = 'Closed "%s" ballot' % e.ballot_type.name
+            e.save()
+
+
+def reverse(apps, schema_editor):
+    pass
+    
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('doc', '0030_fix_bytes_mailarch_url'),
+        ('person', '0009_auto_20190118_0725'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]

ietf/doc/models.py

@@ -332,7 +332,6 @@ class DocumentInfo(models.Model):
                 else:
                     return "Replaced"
             elif state.slug == "active":
-                log.assertion('iesg_state')
                 if iesg_state:
                     if iesg_state.slug == "dead":
                         # Many drafts in the draft-iesg "Dead" state are not dead
@@ -376,7 +375,14 @@ class DocumentInfo(models.Model):
         return self.rfc_number()
 
     def author_list(self):
-        return ", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id)
+        best_addresses = []
+        for author in self.documentauthor_set.all():
+            if author.email:
+                if author.email.active or not author.email.person:
+                    best_addresses.append(author.email.address)
+                else:
+                    best_addresses.append(author.email.person.email_address())
+        return ", ".join(best_addresses)
 
     def authors(self):
         return [ a.person for a in self.documentauthor_set.all() ]

ietf/doc/templatetags/ballot_icon.py

@@ -43,7 +43,6 @@ from django.utils.safestring import mark_safe
 from ietf.ietfauth.utils import user_is_person, has_role
 from ietf.doc.models import BallotPositionDocEvent, IESG_BALLOT_ACTIVE_STATES
 from ietf.name.models import BallotPositionName
-from ietf.utils import log
 
 
 register = template.Library()
@@ -168,7 +167,6 @@ def state_age_colored(doc):
             # Don't show anything for expired/withdrawn/replaced drafts
             return ""
         iesg_state = doc.get_state_slug('draft-iesg')
-        log.assertion('iesg_state')
         if not iesg_state:
             return ""
 

ietf/doc/templatetags/ietf_filters.py

@@ -16,6 +16,7 @@ from django.utils.safestring import mark_safe, SafeData
 from django.utils.html import strip_tags
 from django.utils.encoding import force_text
 from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests
+from django.urls import reverse as urlreverse
 
 import debug                            # pyflakes:ignore
 
@@ -23,6 +24,7 @@ from ietf.doc.models import BallotDocEvent
 from ietf.doc.models import ConsensusDocEvent
 from ietf.utils.html import sanitize_fragment
 from ietf.utils import log
+from ietf.doc.utils import prettify_std_name
 from ietf.utils.text import wordwrap, fill, wrap_text_if_unwrapped
 
 register = template.Library()
@@ -233,6 +235,24 @@ def urlize_ietf_docs(string, autoescape=None):
     return mark_safe(string)
 urlize_ietf_docs = stringfilter(urlize_ietf_docs)
 
+@register.filter(name='urlize_doc_list', is_safe=True, needs_autoescape=True)
+def urlize_doc_list(docs, autoescape=None):
+    """Convert a list of DocAliases into list of links using canonical name"""
+    links = []
+    for doc in docs:
+        name=doc.document.canonical_name()
+        title = doc.document.title
+        url = urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=name))
+        if autoescape:
+            name = escape(name)
+            title = escape(title)
+        links.append(mark_safe(
+            '<a href="%(url)s" title="%(title)s">%(name)s</a>' % dict(name=prettify_std_name(name),
+                                                                      title=title,
+                                                                      url=url)
+        ))
+    return links
+        
 @register.filter(name='dashify')
 def dashify(string):
     """

ietf/doc/tests.py

@@ -514,8 +514,19 @@ Man                    Expires September 22, 2015               [Page 3]
     def test_document_draft(self):
         draft = WgDraftFactory(name='draft-ietf-mars-test',rev='01')
         HolderIprDisclosureFactory(docs=[draft])
+        
+        # Docs for testing relationships. Does not test 'possibly-replaces'. The 'replaced_by' direction
+        # is tested separately below.
         replaced = IndividualDraftFactory()
         draft.relateddocument_set.create(relationship_id='replaces',source=draft,target=replaced.docalias.first())
+        obsoleted = IndividualDraftFactory()
+        draft.relateddocument_set.create(relationship_id='obs',source=draft,target=obsoleted.docalias.first())
+        obsoleted_by = IndividualDraftFactory()
+        obsoleted_by.relateddocument_set.create(relationship_id='obs',source=obsoleted_by,target=draft.docalias.first())
+        updated = IndividualDraftFactory()
+        draft.relateddocument_set.create(relationship_id='updates',source=draft,target=updated.docalias.first())
+        updated_by = IndividualDraftFactory()
+        updated_by.relateddocument_set.create(relationship_id='updates',source=obsoleted_by,target=draft.docalias.first())
 
         # these tests aren't testing all attributes yet, feel free to
         # expand them
@@ -525,24 +536,68 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertContains(r, "Active Internet-Draft")
         self.assertContains(r, "Show full document text")
         self.assertNotContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=0")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "Active Internet-Draft")
         self.assertNotContains(r, "Show full document text")
         self.assertContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=foo")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "Active Internet-Draft")
         self.assertNotContains(r, "Show full document text")
         self.assertContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=1")
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "Active Internet-Draft")
         self.assertNotContains(r, "Show full document text")
         self.assertContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         self.client.cookies = SimpleCookie({str('full_draft'): str('on')})
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
@@ -550,6 +605,17 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertContains(r, "Active Internet-Draft")
         self.assertNotContains(r, "Show full document text")
         self.assertContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         self.client.cookies = SimpleCookie({str('full_draft'): str('off')})
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
@@ -557,6 +623,17 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertContains(r, "Active Internet-Draft")
         self.assertContains(r, "Show full document text")
         self.assertNotContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         self.client.cookies = SimpleCookie({str('full_draft'): str('foo')})
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
@@ -564,6 +641,17 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertContains(r, "Active Internet-Draft")
         self.assertContains(r, "Show full document text")
         self.assertNotContains(r, "Deimos street")
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates not included until draft is RFC
+        self.assertNotContains(r, obsoleted.canonical_name())
+        self.assertNotContains(r, obsoleted.title)
+        self.assertNotContains(r, obsoleted_by.canonical_name())
+        self.assertNotContains(r, obsoleted_by.title)
+        self.assertNotContains(r, updated.canonical_name())
+        self.assertNotContains(r, updated.title)
+        self.assertNotContains(r, updated_by.canonical_name())
+        self.assertNotContains(r, updated_by.title)
 
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name)))
         self.assertEqual(r.status_code, 200)
@@ -602,7 +690,8 @@ Man                    Expires September 22, 2015               [Page 3]
         r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "Replaced Internet-Draft")
-        self.assertContains(r, replacement.name)
+        self.assertContains(r, replacement.canonical_name())
+        self.assertContains(r, replacement.title)
         rel.delete()
 
         # draft published as RFC
@@ -625,6 +714,17 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, "RFC 123456")
         self.assertContains(r, draft.name)
+        self.assertContains(r, replaced.canonical_name())
+        self.assertContains(r, replaced.title)
+        # obs/updates included with RFC
+        self.assertContains(r, obsoleted.canonical_name())
+        self.assertContains(r, obsoleted.title)
+        self.assertContains(r, obsoleted_by.canonical_name())
+        self.assertContains(r, obsoleted_by.title)
+        self.assertContains(r, updated.canonical_name())
+        self.assertContains(r, updated.title)
+        self.assertContains(r, updated_by.canonical_name())
+        self.assertContains(r, updated_by.title)
 
         # naked RFC - also wierd that we test a PS from the ISE
         rfc = IndividualDraftFactory(

ietf/doc/tests_ballot.py

@@ -619,10 +619,10 @@ class BallotWriteupsTests(TestCase):
             verify_can_see(username, url)
 
 class ApproveBallotTests(TestCase):
-    @mock.patch('ietf.sync.rfceditor.urlopen', autospec=True)
+    @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True)
     def test_approve_ballot(self, mock_urlopen):
-        mock_urlopen.return_value.read = lambda : b'OK'
+        mock_urlopen.return_value.text = b'OK'
-        mock_urlopen.return_value.getcode = lambda :200
+        mock_urlopen.return_value.status_code = 200
         #
         ad = Person.objects.get(name="Areað Irector")
         draft = IndividualDraftFactory(ad=ad, intended_std_level_id='ps')

ietf/doc/tests_draft.py

@@ -1197,10 +1197,10 @@ class SubmitToIesgTests(TestCase):
 
 
 class RequestPublicationTests(TestCase):
-    @mock.patch('ietf.sync.rfceditor.urlopen', autospec=True)
+    @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True)
-    def test_request_publication(self, mock_urlopen):
+    def test_request_publication(self, mockobj):
-        mock_urlopen.return_value.read = lambda : b'OK'
+        mockobj.return_value.text = b'OK'
-        mock_urlopen.return_value.getcode = lambda :200
+        mockobj.return_value.status_code = 200
         #
         draft = IndividualDraftFactory(stream_id='iab',group__acronym='iab',intended_std_level_id='inf',states=[('draft-stream-iab','approved')])
 

ietf/doc/utils_charter.py

@@ -15,7 +15,8 @@ from django.utils.encoding import smart_text, force_text
 
 import debug                            # pyflakes:ignore
 
-from ietf.doc.models import NewRevisionDocEvent, WriteupDocEvent 
+from ietf.doc.models import NewRevisionDocEvent, WriteupDocEvent, State, StateDocEvent
+from ietf.doc.utils import close_open_ballots
 from ietf.group.models import ChangeStateGroupEvent
 from ietf.name.models import GroupStateName
 from ietf.utils.history import find_history_active_at
@@ -244,4 +245,18 @@ def generate_issue_ballot_mail(request, doc, ballot):
                                  )
                             )
 
-        
+def replace_charter_of_replaced_group(group, by):
+
+    assert group.state_id == 'replaced'
+
+    charter = group.charter
+
+    if charter:
+
+        close_open_ballots(charter, by)
+
+        replaced_state = State.objects.get(type_id='charter', slug='replaced')
+        charter.set_state(replaced_state)
+        state_change_event = StateDocEvent.objects.create(state_type_id='charter', state=replaced_state, doc=charter, rev=charter.rev, by=by, type="changed_state", desc="Charter's group has been replaced")
+    
+        charter.save_with_history([state_change_event])  

ietf/doc/utils_search.py

@@ -183,7 +183,7 @@ def prepare_document_table(request, docs, query=None, max_results=200):
         else:
             res.append(d.type_id);
             res.append("-");
-            res.append(d.get_state_slug());
+            res.append(d.get_state_slug() or '');
             res.append("-");
 
         if sort_key == "title":

ietf/doc/views_doc.py

@@ -75,7 +75,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import markup_txt, log
+from ietf.utils import markup_txt
 from ietf.utils.text import maybe_split
 
 
@@ -402,7 +402,6 @@ def document_main(request, name, rev=None):
                 actions.append((label, urlreverse('ietf.doc.views_draft.request_publication', kwargs=dict(name=doc.name))))
 
         if doc.get_state_slug() not in ["rfc", "expired"] and doc.stream_id in ("ietf",) and not snapshot:
-            log.assertion('iesg_state')
             if iesg_state.slug == 'idexists' and can_edit:
                 actions.append(("Begin IESG Processing", urlreverse('ietf.doc.views_draft.edit_info', kwargs=dict(name=doc.name)) + "?new=1"))
             elif can_edit_stream_info and (iesg_state.slug in ('idexists','watching')):
@@ -410,10 +409,6 @@ def document_main(request, name, rev=None):
 
         augment_docs_and_user_with_user_info([doc], request.user)
 
-        replaces = [d.name for d in doc.related_that_doc("replaces")]
-        replaced_by = [d.name for d in doc.related_that("replaces")]
-        possibly_replaces = [d.name for d in doc.related_that_doc("possibly-replaces")]
-        possibly_replaced_by = [d.name for d in doc.related_that("possibly-replaces")]
         published = doc.latest_event(type="published_rfc")
         started_iesg_process = doc.latest_event(type="started_iesg_process")
 
@@ -457,14 +452,14 @@ def document_main(request, name, rev=None):
                                        submission=submission,
                                        resurrected_by=resurrected_by,
 
-                                       replaces=replaces,
+                                       replaces=doc.related_that_doc("replaces"),
-                                       replaced_by=replaced_by,
+                                       replaced_by=doc.related_that("replaces"),
-                                       possibly_replaces=possibly_replaces,
+                                       possibly_replaces=doc.related_that_doc("possibly_replaces"),
-                                       possibly_replaced_by=possibly_replaced_by,
+                                       possibly_replaced_by=doc.related_that("possibly_replaces"),
-                                       updates=[prettify_std_name(d.name) for d in doc.related_that_doc("updates")],
+                                       updates=doc.related_that_doc("updates"),
-                                       updated_by=[prettify_std_name(d.document.canonical_name()) for d in doc.related_that("updates")],
+                                       updated_by=doc.related_that("updates"),
-                                       obsoletes=[prettify_std_name(d.name) for d in doc.related_that_doc("obs")],
+                                       obsoletes=doc.related_that_doc("obs"),
-                                       obsoleted_by=[prettify_std_name(d.document.canonical_name()) for d in doc.related_that("obs")],
+                                       obsoleted_by=doc.related_that("obs"),
                                        conflict_reviews=conflict_reviews,
                                        status_changes=status_changes,
                                        proposed_status_changes=proposed_status_changes,
@@ -659,7 +654,7 @@ def document_main(request, name, rev=None):
                            revisions=revisions,
                            latest_rev=latest_rev,
                            snapshot=snapshot,
-                           review_req=review_assignment.review_request,
+                           review_req=review_assignment.review_request if review_assignment else None,
                            other_reviews=other_reviews,
                            assignments=assignments,
                       ))

ietf/doc/views_review.py

@@ -858,7 +858,7 @@ def complete_review(request, name, assignment_id=None, acronym=None):
 
                 list_name = mailarch.list_name_from_email(assignment.review_request.team.list_email)
                 if list_name:
-                    review.external_url = mailarch.construct_message_url(list_name, email.utils.unquote(msg["Message-ID"]))
+                    review.external_url = mailarch.construct_message_url(list_name, email.utils.unquote(msg["Message-ID"].strip()))
                     review.save_with_history([close_event])
 
             if form.cleaned_data['email_ad'] or assignment.result in assignment.review_request.team.reviewteamsettings.notify_ad_when.all(): 

ietf/doc/views_search.py

@@ -352,7 +352,7 @@ def ad_dashboard_sort_key(doc):
         state = State.objects.get(type__slug='draft-iesg',slug='ad-eval')
         return "1%d%s" % (state.order,seed)
 
-    if doc.type.slug=='charter':
+    if doc.type.slug=='charter' and doc.get_state_slug('charter') != 'replaced':
         if doc.get_state_slug('charter') in ('notrev','infrev'):
             return "100%s" % seed
         elif  doc.get_state_slug('charter') == 'intrev':

ietf/group/templatetags/group_filters.py

@@ -1,5 +1,7 @@
 from django import template
 
+import debug                            # pyflakes:ignore
+
 from ietf.group.models import Group
 
 register = template.Library()
@@ -25,3 +27,12 @@ def active_nomcoms(user):
         state__slug='active').distinct().select_related("type"))
 
     return groups
+
+@register.inclusion_tag('person/person_link.html')
+def role_person_link(role, **kwargs):
+    title = kwargs.get('title', '')
+    cls = kwargs.get('class', '')
+    name = role.person.name
+    plain_name = role.person.plain_name()
+    email = role.email.address
+    return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls}

ietf/group/tests_info.py

@@ -23,7 +23,7 @@ from django.utils.html import escape
 
 from ietf.community.models import CommunityList
 from ietf.community.utils import reset_name_contains_index_for_rule
-from ietf.doc.factories import WgDraftFactory, CharterFactory
+from ietf.doc.factories import WgDraftFactory, CharterFactory, BallotDocEventFactory
 from ietf.doc.models import Document, DocAlias, DocEvent, State
 from ietf.doc.utils_charter import charter_name_for_group
 from ietf.group.factories import (GroupFactory, RoleFactory, GroupEventFactory, 
@@ -755,6 +755,19 @@ class GroupEditTests(TestCase):
         group = Group.objects.get(acronym=group.acronym)
         self.assertEqual(group.state_id, "active")
 
+    def test_replace(self):
+        group = GroupFactory(state_id='bof')
+        charter = CharterFactory(group=group, states=[('charter','intrev')])
+        BallotDocEventFactory(doc=charter, ballot_type__doc_type_id='draft', ballot_type__slug='r-extrev')
+        url = urlreverse('ietf.group.views.edit', kwargs=dict(group_type=group.type_id, acronym=group.acronym, action="edit", field="state"))
+        self.client.login(username='secretary',password='secretary+password')
+        self.client.post(url, dict(state='replaced'))
+        group = Group.objects.get(pk=group.pk)       
+        self.assertEqual(group.state_id, 'replaced')
+        self.assertEqual(group.charter.get_state_slug('charter'), 'replaced')
+        self.assertEqual(group.charter.active_ballot(), None)
+
+
     def test_add_comment(self):
         group = GroupFactory(acronym="mars",parent=GroupFactory(type_id='area'))
         RoleFactory(group=group,person=Person.objects.get(user__username='ad'),name_id='ad')

ietf/group/utils.py

@@ -245,7 +245,7 @@ def construct_group_menu_context(request, group, selected, group_type, others):
     if group.features.customize_workflow and can_manage:
         actions.append(("Customize workflow", urlreverse("ietf.group.views.customize_workflow", kwargs=kwargs)))
 
-    if group.state_id in ("active", "dormant") and not group.type_id in ["sdo", "rfcedtyp", "isoc", ] and can_manage_group_type(request.user, group):
+    if group.state_id in ("active", "dormant") and group.type_id in ["wg", "rg", ] and can_manage_group_type(request.user, group):
         actions.append(("Request closing group", urlreverse("ietf.group.views.conclude", kwargs=kwargs)))
 
     d = {

ietf/group/views.py

@@ -65,7 +65,7 @@ from ietf.community.utils import docs_tracked_by_community_list
 from ietf.doc.models import DocTagName, State, DocAlias, RelatedDocument, Document
 from ietf.doc.templatetags.ietf_filters import clean_whitespace
 from ietf.doc.utils import get_chartering_type, get_tags_for_stream_id
-from ietf.doc.utils_charter import charter_name_for_group
+from ietf.doc.utils_charter import charter_name_for_group, replace_charter_of_replaced_group
 from ietf.doc.utils_search import prepare_document_table
 #
 from ietf.group.dot import make_dot
@@ -1026,6 +1026,8 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None):
                 for attr, new, desc in changes:
                     if attr == 'state':
                         ChangeStateGroupEvent.objects.create(group=group, time=group.time, state=new, by=request.user.person, type="changed_state", desc=desc)
+                        if new.slug == 'replaced':
+                            replace_charter_of_replaced_group(group=group, by=request.user.person)
                     else:
                         GroupEvent.objects.create(group=group, time=group.time, by=request.user.person, type="info_changed", desc=desc)
 

ietf/idindex/index.py

@@ -20,7 +20,6 @@ from ietf.doc.models import IESG_SUBSTATE_TAGS
 from ietf.doc.templatetags.ietf_filters import clean_whitespace
 from ietf.group.models import Group
 from ietf.person.models import Person, Email
-from ietf.utils import log
 
 def all_id_txt():
     # this returns a lot of data so try to be efficient
@@ -154,7 +153,6 @@ def all_id2_txt():
         # 3
         if state == "active":
             s = "I-D Exists"
-            log.assertion('iesg_state')
             if iesg_state:
                 s = iesg_state.name
                 tags = d.tags.filter(slug__in=IESG_SUBSTATE_TAGS).values_list("name", flat=True)

ietf/ipr/forms.py

@@ -111,18 +111,21 @@ class DraftForm(forms.ModelForm):
         }
         help_texts = { 'sections': 'Sections' }
 
+patent_number_help_text = "Enter one or more comma-separated patent publication or application numbers as two-letter country code and serial number, e.g.: US62/123456 or WO2017123456. Do not include thousands-separator commas in serial numbers.  It is preferable to use individual disclosures for each patent, even if this field permits multiple patents to be listed, in order to get inventor, title, and date information below correct." 
 validate_patent_number = RegexValidator(
                                     regex=(r"^("
-                                             r"([A-Z][A-Z]\d\d/\d{6}"
+                                             r"([A-Z][A-Z] *\d\d/\d{6}"
-                                             r"|[A-Z][A-Z]\d{6,12}([A-Z]\d?)?"
+                                             r"|[A-Z][A-Z] *\d{6,12}([A-Z]\d?)?"
-                                             r"|[A-Z][A-Z]\d{4}(\w{1,2}\d{5,7})?"
+                                             r"|[A-Z][A-Z] *\d{4}(\w{1,2}\d{5,7})?"
-                                             r"|[A-Z][A-Z]\d{15}"
+                                             r"|[A-Z][A-Z] *\d{15}"
-                                             r"|[A-Z][A-Z][A-Z]\d{1,5}/\d{4}" 
+                                             r"|[A-Z][A-Z][A-Z] *\d{1,5}/\d{4}" 
-                                             r"|[A-Z][A-Z]\d{1,4}/\d{1,4}" 
+                                             r"|[A-Z][A-Z] *\d{1,4}/\d{1,4}" 
-                                             r"|PCT/[A-Z][A-Z]\d{2}/\d{5}" # WO application, old
+                                             r"|PCT/[A-Z][A-Z]*\d{2}/\d{5}" # WO application, old
-                                             r"|PCT/[A-Z][A-Z]\d{4}/\d{6}" # WO application, new
+                                             r"|PCT/[A-Z][A-Z]*\d{4}/\d{6}" # WO application, new
                                              r")[, ]*)+$"),
-                                    message="Please enter one or more patent publication or application numbers as country code and serial number, e.g.: US62/123456 or WO2017123456." )
+                                    message=patent_number_help_text)
+
+
 
 """
 Patent application number formats by country
@@ -206,7 +209,7 @@ class GenericDisclosureForm(forms.Form):
     submitter_email = forms.EmailField(required=False)
     #patent_info = forms.CharField(max_length=255,widget=forms.Textarea, required=False, help_text="Patent, Serial, Publication, Registration, or Application/File number(s), Date(s) granted or applied for, Country, and any additional notes.", strip=False)
     patent_number = forms.CharField(max_length=127, required=False, validators=[ validate_patent_number ],
-        help_text = "Patent publication or application number (2-letter country code followed by serial number)")
+        help_text = patent_number_help_text)
     patent_inventor =  forms.CharField(max_length=63, required=False, validators=[ validate_name ], help_text="Inventor name")
     patent_title =  forms.CharField(max_length=255, required=False, validators=[ validate_title ], help_text="Title of invention")
     patent_date =  forms.DateField(required=False, help_text="Date granted or applied for")
@@ -275,7 +278,7 @@ class IprDisclosureFormBase(forms.ModelForm):
     updates = SearchableIprDisclosuresField(required=False, help_text=mark_safe("If this disclosure <strong>updates</strong> other disclosures identify here which ones. Leave this field blank if this disclosure does not update any prior disclosures. Note: Updates to IPR disclosures must only be made by authorized representatives of the original submitters. Updates will automatically be forwarded to the current Patent Holder's Contact and to the Submitter of the original IPR disclosure."))
     same_as_ii_above = forms.BooleanField(required=False)
     patent_number = forms.CharField(max_length=127, required=True, validators=[ validate_patent_number ],
-        help_text = "Patent publication or application number (2-letter country code followed by serial number)")
+        help_text = patent_number_help_text)
     patent_inventor =  forms.CharField(max_length=63, required=True, validators=[ validate_name ], help_text="Inventor name")
     patent_title =  forms.CharField(max_length=255, required=True, validators=[ validate_title ], help_text="Title of invention")
     patent_date =  forms.DateField(required=True, help_text="Date granted or applied for")

ietf/liaisons/widgets.py

@@ -36,7 +36,7 @@ class ShowAttachmentsWidget(Widget):
         html += '<div class="attachedFiles form-control widget">'
         if value and isinstance(value, QuerySet):
             for attachment in value:
-                html += '<a class="initialAttach" href="%s">%s</a>&nbsp' % (conditional_escape(attachment.document.href()), conditional_escape(attachment.document.title))
+                html += '<a class="initialAttach" href="%s">%s</a>&nbsp' % (conditional_escape(attachment.document.get_href()), conditional_escape(attachment.document.title))
                 html += '<a class="btn btn-default btn-xs" href="{}">Edit</a>&nbsp'.format(urlreverse("ietf.liaisons.views.liaison_edit_attachment", kwargs={'object_id':attachment.statement.pk,'doc_id':attachment.document.pk}))
                 html += '<a class="btn btn-default btn-xs" href="{}">Delete</a>&nbsp'.format(urlreverse("ietf.liaisons.views.liaison_delete_attachment", kwargs={'object_id':attachment.statement.pk,'attach_id':attachment.pk}))
                 html += '<br />'

ietf/meeting/forms.py

@@ -101,7 +101,7 @@ class InterimSessionInlineFormSet(BaseInlineFormSet):
 
 class InterimMeetingModelForm(forms.ModelForm):
     # TODO: Should area groups get to schedule Interims?
-    group = GroupModelChoiceField(queryset=Group.objects.filter(type__in=('wg', 'rg'), state__in=('active', 'proposed', 'bof')).order_by('acronym'), required=False)
+    group = GroupModelChoiceField(queryset=Group.objects.filter(type__in=('wg', 'rg', 'ag'), state__in=('active', 'proposed', 'bof')).order_by('acronym'), required=False)
     in_person = forms.BooleanField(required=False)
     meeting_type = forms.ChoiceField(choices=(
         ("single", "Single"),
@@ -216,8 +216,8 @@ class InterimSessionModelForm(forms.ModelForm):
             self.user = kwargs.pop('user')
         if 'group' in kwargs:
             self.group = kwargs.pop('group')
-        if 'is_approved_or_virtual' in kwargs:
+        if 'requires_approval' in kwargs:
-            self.is_approved_or_virtual = kwargs.pop('is_approved_or_virtual')
+            self.requires_approval = kwargs.pop('requires_approval')
         super(InterimSessionModelForm, self).__init__(*args, **kwargs)
         self.is_edit = bool(self.instance.pk)
         # setup fields that aren't intrinsic to the Session object
@@ -238,6 +238,14 @@ class InterimSessionModelForm(forms.ModelForm):
             raise forms.ValidationError('Required field')
         return date
 
+    def clean_requested_duration(self):
+        min_minutes = settings.INTERIM_SESSION_MINIMUM_MINUTES
+        max_minutes = settings.INTERIM_SESSION_MAXIMUM_MINUTES
+        duration = self.cleaned_data.get('requested_duration')
+        if not duration or duration < datetime.timedelta(minutes=min_minutes) or duration > datetime.timedelta(minutes=max_minutes):
+            raise forms.ValidationError('Provide a duration, %s-%smin.' % (min_minutes, max_minutes))
+        return duration
+
     def save(self, *args, **kwargs):
         """NOTE: as the baseform of an inlineformset self.save(commit=True)
         never gets called"""

ietf/meeting/helpers.py

@@ -221,7 +221,7 @@ def read_session_file(type, num, doc):
     #
     # FIXME: uploaded_filename should be replaced with a function call that computes names that are fixed
     path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.uploaded_filename))
-    if os.path.exists(path):
+    if doc.uploaded_filename and os.path.exists(path):
         with io.open(path, 'rb') as f:
             return f.read(), path
     else:
@@ -324,8 +324,9 @@ def can_approve_interim_request(meeting, user):
     if not session:
         return False
     group = session.group
-    if group.type.slug == 'wg' and group.parent.role_set.filter(name='ad', person=person):
+    if group.type.slug == 'wg':
-        return True
+        if group.parent.role_set.filter(name='ad', person=person) or group.role_set.filter(name='ad', person=person):
+            return True
     if group.type.slug == 'rg' and group.parent.role_set.filter(name='chair', person=person):
         return True
     return False
@@ -600,7 +601,7 @@ def sessions_post_save(request, forms):
             continue
 
         if form.instance.pk is not None and not SchedulingEvent.objects.filter(session=form.instance).exists():
-            if form.is_approved_or_virtual:
+            if not form.requires_approval:
                 status_id = 'scheda'
             else:
                 status_id = 'apprw'

ietf/meeting/migrations/0026_cancel_107_sessions.py

@@ -0,0 +1,40 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2020-03-18 16:18
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def cancel_sessions(apps, schema_editor):
+    Session = apps.get_model('meeting', 'Session')
+    SchedulingEvent = apps.get_model('meeting', 'SchedulingEvent')
+    SessionStatusName = apps.get_model('name', 'SessionStatusName')
+    Person = apps.get_model('person', 'Person')
+    excludes = ['txauth','dispatch','add','raw','masque','wpack','drip','gendispatch','privacypass', 'ript', 'secdispatch', 'webtrans']
+    canceled = SessionStatusName.objects.get(slug='canceled')
+    person = Person.objects.get(name='Ryan Cross')
+    sessions = Session.objects.filter(meeting__number=107,group__type__in=['wg','rg','ag']).exclude(group__acronym__in=excludes)
+    for session in sessions:
+        SchedulingEvent.objects.create(
+            session = session,
+            status = canceled,
+            by = person)
+
+
+def reverse(apps, schema_editor):
+    SchedulingEvent = apps.get_model('meeting', 'SchedulingEvent')
+    Person = apps.get_model('person', 'Person')
+    person = Person.objects.get(name='Ryan Cross')
+    SchedulingEvent.objects.filter(meeting__number=107, by=person).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('meeting', '0025_rename_type_session_to_regular'),
+    ]
+
+    operations = [
+        migrations.RunPython(cancel_sessions, reverse),
+    ]

ietf/meeting/models.py

@@ -400,6 +400,9 @@ class Room(models.Model):
     def video_stream_url(self):
         urlresource = self.urlresource_set.filter(name_id__in=['meetecho', ]).first()
         return urlresource.url if urlresource else None
+    def webex_url(self):
+        urlresource = self.urlresource_set.filter(name_id__in=['webex', ]).first()
+        return urlresource.url if urlresource else None
     #
     class Meta:
         ordering = ["-id"]
@@ -781,13 +784,15 @@ class SchedTimeSessAssignment(models.Model):
         """Return sensible id string for session, e.g. suitable for use as HTML anchor."""
         components = []
 
+        components.append(self.schedule.meeting.number)
+
         if not self.timeslot:
             components.append("unknown")
 
         if not self.session or not (getattr(self.session, "historic_group") or self.session.group):
             components.append("unknown")
         else:
-            components.append(self.timeslot.time.strftime("%a-%H%M"))
+            components.append(self.timeslot.time.strftime("%Y-%m-%d-%a-%H%M"))
 
             g = getattr(self.session, "historic_group", None) or self.session.group
 

ietf/meeting/tests_views.py

@@ -4,6 +4,7 @@
 
 import datetime
 import io
+import json
 import os
 import random
 import re
@@ -19,7 +20,7 @@ from urllib.parse import urlparse
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django.contrib.auth.models import User
-from django.test import Client
+from django.test import Client, override_settings
 from django.db.models import F
 
 import debug           # pyflakes:ignore
@@ -35,11 +36,10 @@ from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignm
 from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting
 from ietf.meeting.utils import finalize, condition_slide_order
 from ietf.meeting.utils import add_event_info_to_session_qs
-from ietf.meeting.utils import current_session_status
 from ietf.meeting.views import session_draft_list
 from ietf.name.models import SessionStatusName, ImportantDateName
 from ietf.utils.decorators import skip_coverage
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import outbox, empty_outbox, get_payload
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.utils.text import xslugify
 
@@ -433,9 +433,10 @@ class MeetingTests(TestCase):
         response = self.client.get(url)
         self.assertContains(response, 'test acknowledgements')
 
-    @patch('urllib.request.urlopen')
+    @patch('ietf.meeting.utils.requests.get')
-    def test_proceedings_attendees(self, mock_urlopen):
+    def test_proceedings_attendees(self, mockobj):
-        mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
+        mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
+        mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96")
         finalize(meeting)
@@ -602,6 +603,14 @@ class MeetingTests(TestCase):
         self.assertEqual(response.status_code,200)
         self.assertEqual(response.get('Content-Type'), 'text/calendar')
 
+    def test_cancelled_ics(self):
+        session=SessionFactory(meeting__type_id='ietf',status_id='canceled')
+        url = urlreverse('ietf.meeting.views.ical_agenda', kwargs=dict(num=session.meeting.number))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        self.assertIn('STATUS:CANCELLED',unicontent(r))
+        self.assertNotIn('STATUS:CONFIRMED',unicontent(r))
+
 class ReorderSlidesTests(TestCase):
 
     def test_add_slides_to_session(self):
@@ -1081,6 +1090,24 @@ class SessionDetailsTests(TestCase):
         r = self.client.get(url)
         self.assertTrue(all([x in unicontent(r) for x in ('slides','agenda','minutes','draft')]))
         self.assertNotContains(r, 'deleted')
+
+    def test_session_details_past_interim(self):
+        group = GroupFactory.create(type_id='wg',state_id='active')
+        chair = RoleFactory(name_id='chair',group=group)
+        session = SessionFactory.create(meeting__type_id='interim',group=group, meeting__date=datetime.date.today()-datetime.timedelta(days=90))
+        SessionPresentationFactory.create(session=session,document__type_id='draft',rev=None)
+        SessionPresentationFactory.create(session=session,document__type_id='minutes')
+        SessionPresentationFactory.create(session=session,document__type_id='slides')
+        SessionPresentationFactory.create(session=session,document__type_id='agenda')
+
+        url = urlreverse('ietf.meeting.views.session_details', kwargs=dict(num=session.meeting.number, acronym=group.acronym))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        self.assertNotIn('The materials upload cutoff date for this session has passed', unicontent(r))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        self.client.login(username=chair.person.user.username,password=chair.person.user.username+'+password')
+        self.assertTrue(all([x in unicontent(r) for x in ('slides','agenda','minutes','draft')]))
         
     def test_add_session_drafts(self):
         group = GroupFactory.create(type_id='wg',state_id='active')
@@ -1248,6 +1275,8 @@ class InterimTests(TestCase):
     def test_interim_send_announcement(self):
         make_meeting_test_data()
         meeting = add_event_info_to_session_qs(Session.objects.filter(meeting__type='interim', group__acronym='mars')).filter(current_status='apprw').first().meeting
+        meeting.time_zone = 'America/Los_Angeles'
+        meeting.save()
         url = urlreverse("ietf.meeting.views.interim_send_announcement", kwargs={'number': meeting.number})
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
@@ -1259,6 +1288,8 @@ class InterimTests(TestCase):
         self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_announce'))
         self.assertEqual(len(outbox), len_before + 1)
         self.assertIn('WG Virtual Meeting', outbox[-1]['Subject'])
+        self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload(outbox[-1]))
+        self.assertIn('(17:00 to 17:20 UTC)', get_payload(outbox[-1]))
 
     def test_interim_approve_by_ad(self):
         make_meeting_test_data()
@@ -1287,13 +1318,14 @@ class InterimTests(TestCase):
         today = datetime.date.today()
         last_week = today - datetime.timedelta(days=7)
         ietf = SessionFactory(meeting__type_id='ietf',meeting__date=last_week,group__state_id='active',group__parent=GroupFactory(state_id='active'))
-        interim = SessionFactory(meeting__type_id='interim',meeting__date=last_week,status_id='canceled',group__state_id='active',group__parent=GroupFactory(state_id='active'))
+        SessionFactory(meeting__type_id='interim',meeting__date=last_week,status_id='canceled',group__state_id='active',group__parent=GroupFactory(state_id='active'))
         url = urlreverse('ietf.meeting.views.past')
         r = self.client.get(url)
         self.assertContains(r, 'IETF - %02d'%int(ietf.meeting.number))
         q = PyQuery(r.content)
-        id="-%s" % interim.group.acronym
+        #id="-%s" % interim.group.acronym
-        self.assertIn('CANCELLED', q('[id*="'+id+'"]').text())
+        #self.assertIn('CANCELLED', q('[id*="'+id+'"]').text())
+        self.assertIn('CANCELLED', q('tr>td>a>span').text())
 
     def test_upcoming(self):
         make_meeting_test_data()
@@ -1305,10 +1337,11 @@ class InterimTests(TestCase):
         r = self.client.get(url)
         self.assertContains(r, mars_interim.number)
         self.assertContains(r, ames_interim.number)
-        self.assertContains(r, 'IETF - 72')
+        self.assertContains(r, 'IETF 72')
         # cancelled session
         q = PyQuery(r.content)
-        self.assertIn('CANCELLED', q('[id*="-ames"]').text())
+#        self.assertIn('CANCELLED', q('[id*="-ames"]').text())
+        self.assertIn('CANCELLED', q('tr>td>a>span').text())
         self.check_interim_tabs(url)
 
     def test_upcoming_ical(self):
@@ -1362,7 +1395,7 @@ class InterimTests(TestCase):
         r = self.client.get("/meeting/interim/request/")
         self.assertEqual(r.status_code, 200)
         q = PyQuery(r.content)
-        self.assertEqual(Group.objects.filter(type__in=('wg', 'rg'), state__in=('active', 'proposed')).count(),
+        self.assertEqual(Group.objects.filter(type__in=('wg', 'rg', 'ag'), state__in=('active', 'proposed')).count(),
             len(q("#id_group option")) - 1)  # -1 for options placeholder
         self.client.logout()
 
@@ -1385,7 +1418,7 @@ class InterimTests(TestCase):
         count = person.role_set.filter(name='chair',group__type__in=('wg', 'rg'), group__state__in=('active', 'proposed')).count()
         self.assertEqual(count, len(q("#id_group option")) - 1)  # -1 for options placeholder
 
-    def test_interim_request_single_virtual(self):
+    def do_interim_request_single_virtual(self):
         make_meeting_test_data()
         group = Group.objects.get(acronym='mars')
         date = datetime.date.today() + datetime.timedelta(days=30)
@@ -1427,7 +1460,6 @@ class InterimTests(TestCase):
         session = meeting.session_set.first()
         self.assertEqual(session.remote_instructions,remote_instructions)
         self.assertEqual(session.agenda_note,agenda_note)
-        self.assertEqual(current_session_status(session).slug,'scheda')
         timeslot = session.official_timeslotassignment().timeslot
         self.assertEqual(timeslot.time,dt)
         self.assertEqual(timeslot.duration,duration)
@@ -1438,8 +1470,22 @@ class InterimTests(TestCase):
         self.assertTrue(os.path.exists(path))
         # check notice to secretariat
         self.assertEqual(len(outbox), length_before + 1)
-        self.assertIn('interim meeting ready for announcement', outbox[-1]['Subject'])
+        return meeting
+
+    @override_settings(VIRTUAL_INTERIMS_REQUIRE_APPROVAL = True)
+    def test_interim_request_single_virtual_settings_approval_required(self):
+        meeting = self.do_interim_request_single_virtual()
+        self.assertEqual(meeting.session_set.last().schedulingevent_set.last().status_id,'apprw')
+        self.assertIn('New Interim Meeting Request', outbox[-1]['Subject'])
+        self.assertIn('session-request@ietf.org', outbox[-1]['To'])
+        self.assertIn('aread@example.org', outbox[-1]['Cc'])
+
+    @override_settings(VIRTUAL_INTERIMS_REQUIRE_APPROVAL = False)
+    def test_interim_request_single_virtual_settings_approval_not_required(self):
+        meeting = self.do_interim_request_single_virtual()
+        self.assertEqual(meeting.session_set.last().schedulingevent_set.last().status_id,'scheda')
         self.assertIn('iesg-secretary@ietf.org', outbox[-1]['To'])
+        self.assertIn('interim meeting ready for announcement', outbox[-1]['Subject'])
 
     def test_interim_request_single_in_person(self):
         make_meeting_test_data()
@@ -1700,9 +1746,12 @@ class InterimTests(TestCase):
         # related AD
         user = User.objects.get(username='ad')
         self.assertTrue(can_approve_interim_request(meeting=meeting,user=user))
-        # other AD
+        # AD from other area
         user = User.objects.get(username='ops-ad')
         self.assertFalse(can_approve_interim_request(meeting=meeting,user=user))
+        # AD from other area assigned as the WG AD anyhow (cross-area AD)
+        user = RoleFactory(name_id='ad',group=group).person.user
+        self.assertTrue(can_approve_interim_request(meeting=meeting,user=user))
         # WG Chair
         user = User.objects.get(username='marschairman')
         self.assertFalse(can_approve_interim_request(meeting=meeting,user=user))
@@ -1836,7 +1885,7 @@ class InterimTests(TestCase):
                 'session_set-0-id':meeting.session_set.first().id,
                 'session_set-0-date':formset_initial['date'].strftime('%Y-%m-%d'),
                 'session_set-0-time':new_time.strftime('%H:%M'),
-                'session_set-0-requested_duration':formset_initial['requested_duration'],
+                'session_set-0-requested_duration': '00:30',
                 'session_set-0-remote_instructions':formset_initial['remote_instructions'],
                 #'session_set-0-agenda':formset_initial['agenda'],
                 'session_set-0-agenda_note':formset_initial['agenda_note'],
@@ -2022,6 +2071,20 @@ class IphoneAppJsonTests(TestCase):
     def tearDown(self):
         pass
 
+    def test_iphone_app_json_interim(self):
+        make_meeting_test_data()
+        meeting = Meeting.objects.filter(type_id='interim').order_by('id').last()
+        url = urlreverse('ietf.meeting.views.json_agenda',kwargs={'num':meeting.number})
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        data = r.json()
+        self.assertIn(meeting.number, data.keys())
+        jsessions = [ s for s in data[meeting.number] if s['objtype'] == 'session' ]
+        msessions = meeting.session_set.exclude(type__in=['lead','offagenda','break','reg'])
+        self.assertEqual(len(jsessions), msessions.count())
+        for s in jsessions:
+            self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
+
     def test_iphone_app_json(self):
         make_meeting_test_data()
         meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last()
@@ -2036,6 +2099,13 @@ class IphoneAppJsonTests(TestCase):
         url = urlreverse('ietf.meeting.views.json_agenda',kwargs={'num':meeting.number})
         r = self.client.get(url)
         self.assertEqual(r.status_code,200)
+        data = r.json()
+        self.assertIn(meeting.number, data.keys())
+        jsessions = [ s for s in data[meeting.number] if s['objtype'] == 'session' ]
+        msessions = meeting.session_set.exclude(type__in=['lead','offagenda','break','reg'])
+        self.assertEqual(len(jsessions), msessions.count())
+        for s in jsessions:
+            self.assertTrue(msessions.filter(group__acronym=s['group']['acronym']).exists())
 
 class FinalizeProceedingsTests(TestCase):
     @patch('urllib.request.urlopen')

ietf/meeting/urls.py

@@ -66,9 +66,10 @@ type_ietf_only_patterns = [
 
 # This is a limited subset of the list above -- many of the views above won't work for interim meetings
 type_interim_patterns = [
-    url(r'^agenda/(?P<session>[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf),
+    url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf),
-    url(r'^agenda/(?P<session>[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile),
+    url(r'^agenda/(?P<acronym>[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile),
     url(r'^materials/%(document)s((?P<ext>\.[a-z0-9]+)|/)?$' % settings.URL_REGEXPS, views.materials_document),
+    url(r'^agenda.json$', views.json_agenda)
 ]
 
 type_ietf_only_patterns_id_optional = [

ietf/meeting/utils.py

@@ -3,8 +3,7 @@
 
 
 import datetime
-import json
+import requests
-import urllib.request
 
 from urllib.error import HTTPError
 from django.conf import settings
@@ -113,7 +112,7 @@ def create_proceedings_templates(meeting):
     # Get meeting attendees from registration system
     url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number)
     try:
-        attendees = json.load(urllib.request.urlopen(url))
+        attendees = requests.get(url).json()
     except (ValueError, HTTPError):
         attendees = []
 
@@ -240,6 +239,26 @@ def only_sessions_that_can_meet(session_qs):
 
     return qs
 
+
+# Keeping this as a note that might help when returning Customization to the /meetings/upcoming page
+#def group_parents_from_sessions(sessions):
+#    group_parents = list()
+#    parents = {}
+#    for s in sessions:
+#        if s.group.parent_id not in parents:
+#            parent = s.group.parent
+#            parent.group_list = set()
+#            group_parents.append(parent)
+#            parents[s.group.parent_id] = parent
+#        parent.group_list.add(s.group)
+#
+#    for p in parents.values():
+#        p.group_list = list(p.group_list)
+#        p.group_list.sort(key=lambda g: g.acronym)
+#
+#    return group_parents
+
+
 def data_for_meetings_overview(meetings, interim_status=None):
     """Return filtered meetings with sessions and group hierarchy (for the
     interim menu)."""
@@ -275,30 +294,12 @@ def data_for_meetings_overview(meetings, interim_status=None):
             if not m.type_id == 'interim' or not all(s.current_status in ['apprw', 'scheda', 'canceledpa'] for s in m.sessions)
         ]
 
-    # group hierarchy
     ietf_group = Group.objects.get(acronym='ietf')
 
-    group_hierarchy = [ietf_group]
-
-    parents = {}
-    for m in meetings:
-        if m.type_id == 'interim' and m.sessions:
-            for s in m.sessions:
-                parent = parents.get(s.group.parent_id)
-                if not parent:
-                    parent = s.group.parent
-                    parent.group_list = []
-                    group_hierarchy.append(parent)
-
-                parent.group_list.append(s.group)
-
-    for p in parents.values():
-        p.group_list.sort(key=lambda g: g.acronym)
-
     # set some useful attributes
     for m in meetings:
         m.end = m.date + datetime.timedelta(days=m.days)
         m.responsible_group = (m.sessions[0].group if m.sessions else None) if m.type_id == 'interim' else ietf_group
         m.interim_meeting_cancelled = m.type_id == 'interim' and all(s.current_status == 'canceled' for s in m.sessions)
 
-    return meetings, group_hierarchy
+    return meetings

ietf/meeting/views.py

@@ -949,7 +949,7 @@ def ical_agenda(request, num=None, name=None, acronym=None, session_id=None):
 
     for a in assignments:
         if a.session:
-            a.session.ical_status = ical_session_status(a.session)
+            a.session.ical_status = ical_session_status(a.session.current_status)
 
     return render(request, "meeting/agenda.ics", {
         "schedule": schedule,
@@ -959,7 +959,7 @@ def ical_agenda(request, num=None, name=None, acronym=None, session_id=None):
 
 @cache_page(15 * 60)
 def json_agenda(request, num=None ):
-    meeting = get_meeting(num)
+    meeting = get_meeting(num, type_in=['ietf','interim'])
 
     sessions = []
     locations = set()
@@ -1156,6 +1156,7 @@ def session_details(request, num, acronym):
     # we somewhat arbitrarily use the group of the last session we get from
     # get_sessions() above when checking can_manage_session_materials()
     can_manage = can_manage_session_materials(request.user, session.group, session)
+    can_view_request = can_view_interim_request(meeting, request.user)
 
     scheduled_sessions = [s for s in sessions if s.current_status == 'sched']
     unscheduled_sessions = [s for s in sessions if s.current_status != 'sched']
@@ -1173,7 +1174,9 @@ def session_details(request, num, acronym):
                     'pending_suggestions' : pending_suggestions,
                     'meeting' :meeting ,
                     'acronym' :acronym,
+                    'is_materials_manager' : session.group.has_role(request.user, session.group.features.matman_roles),
                     'can_manage_materials' : can_manage,
+                    'can_view_request': can_view_request,
                     'thisweek': datetime.date.today()-datetime.timedelta(days=7),
                   })
 
@@ -1916,7 +1919,7 @@ def ajax_get_utc(request):
 @role_required('Secretariat',)
 def interim_announce(request):
     '''View which shows interim meeting requests awaiting announcement'''
-    meetings, _ = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='scheda')
+    meetings = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='scheda')
     menu_entries = get_interim_menu_entries(request)
     selected_menu_entry = 'announce'
 
@@ -1979,7 +1982,7 @@ def interim_skip_announcement(request, number):
 @role_required('Area Director', 'Secretariat', 'IRTF Chair', 'WG Chair', 'RG Chair')
 def interim_pending(request):
     '''View which shows interim meeting requests pending approval'''
-    meetings, group_parents = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='apprw')
+    meetings = data_for_meetings_overview(Meeting.objects.filter(type='interim').order_by('date'), interim_status='apprw')
 
     menu_entries = get_interim_menu_entries(request)
     selected_menu_entry = 'pending'
@@ -2014,6 +2017,8 @@ def interim_request(request):
             is_virtual = form.is_virtual()
             meeting_type = form.cleaned_data.get('meeting_type')
 
+            requires_approval = not ( is_approved or ( is_virtual and not settings.VIRTUAL_INTERIMS_REQUIRE_APPROVAL ))
+
             # pre create meeting
             if meeting_type in ('single', 'multi-day'):
                 meeting = form.save(date=get_earliest_session_date(formset))
@@ -2023,13 +2028,13 @@ def interim_request(request):
                     InterimSessionModelForm.__init__,
                     user=request.user,
                     group=group,
-                    is_approved_or_virtual=(is_approved or is_virtual))
+                    requires_approval=requires_approval)
                 formset = SessionFormset(instance=meeting, data=request.POST)
                 formset.is_valid()
                 formset.save()
                 sessions_post_save(request, formset)
 
-                if not (is_approved or is_virtual):
+                if requires_approval:
                     send_interim_approval_request(meetings=[meeting])
                 elif not has_role(request.user, 'Secretariat'):
                     send_interim_announcement_request(meeting=meeting)
@@ -2043,7 +2048,7 @@ def interim_request(request):
                     InterimSessionModelForm.__init__,
                     user=request.user,
                     group=group,
-                    is_approved_or_virtual=(is_approved or is_virtual))
+                    requires_approval=requires_approval)
                 formset = SessionFormset(instance=Meeting(), data=request.POST)
                 formset.is_valid()  # re-validate
                 for session_form in formset.forms:
@@ -2060,7 +2065,7 @@ def interim_request(request):
                     series.append(meeting)
                     sessions_post_save(request, [session_form])
 
-                if not (is_approved or is_virtual):
+                if requires_approval:
                     send_interim_approval_request(meetings=series)
                 elif not has_role(request.user, 'Secretariat'):
                     send_interim_announcement_request(meeting=meeting)
@@ -2188,7 +2193,7 @@ def interim_request_edit(request, number):
             InterimSessionModelForm.__init__,
             user=request.user,
             group=group,
-            is_approved_or_virtual=is_approved)
+            requires_approval= not is_approved)
 
         formset = SessionFormset(instance=meeting, data=request.POST)
 
@@ -2219,17 +2224,33 @@ def past(request):
     '''List of past meetings'''
     today = datetime.datetime.today()
 
-    meetings, group_parents = data_for_meetings_overview(Meeting.objects.filter(date__lte=today).order_by('-date'))
+    meetings = data_for_meetings_overview(Meeting.objects.filter(date__lte=today).order_by('-date'))
 
     return render(request, 'meeting/past.html', {
                   'meetings': meetings,
-                  'group_parents': group_parents})
+                  })
 
 def upcoming(request):
     '''List of upcoming meetings'''
-    today = datetime.datetime.today()
+    today = datetime.date.today()
 
-    meetings, group_parents = data_for_meetings_overview(Meeting.objects.filter(date__gte=today).order_by('date'))
+    # Get ietf meetings starting 7 days ago, and interim meetings starting today
+    ietf_meetings = Meeting.objects.filter(type_id='ietf', date__gte=today-datetime.timedelta(days=7))
+    for m in ietf_meetings:
+        m.end = m.date+datetime.timedelta(days=m.days)
+    interim_sessions = add_event_info_to_session_qs(
+        Session.objects.filter(
+            meeting__type_id='interim', 
+            timeslotassignments__schedule=F('meeting__schedule'),
+            timeslotassignments__timeslot__time__gte=today
+        )
+    ).filter(current_status__in=('sched','canceled'))
+    for session in interim_sessions:
+        session.historic_group = session.group
+
+    entries = list(ietf_meetings)
+    entries.extend(list(interim_sessions))
+    entries.sort(key = lambda o: pytz.utc.localize(datetime.datetime.combine(o.date, datetime.datetime.min.time())) if isinstance(o,Meeting) else o.official_timeslotassignment().timeslot.utc_start_time())
 
     # add menu entries
     menu_entries = get_interim_menu_entries(request)
@@ -2244,23 +2265,25 @@ def upcoming(request):
                     reverse('ietf.meeting.views.upcoming_ical')))
 
     return render(request, 'meeting/upcoming.html', {
-                  'meetings': meetings,
+                  'entries': entries,
                   'menu_actions': actions,
                   'menu_entries': menu_entries,
                   'selected_menu_entry': selected_menu_entry,
-                  'group_parents': group_parents})
+                  })
 
 
 def upcoming_ical(request):
     '''Return Upcoming meetings in iCalendar file'''
     filters = request.GET.getlist('filters')
-    today = datetime.datetime.today()
+    today = datetime.date.today()
 
-    meetings, _ = data_for_meetings_overview(Meeting.objects.filter(date__gte=today).order_by('date'))
+    # get meetings starting 7 days ago -- we'll filter out sessions in the past further down
+    meetings = data_for_meetings_overview(Meeting.objects.filter(date__gte=today-datetime.timedelta(days=7)).order_by('date'))
 
     assignments = list(SchedTimeSessAssignment.objects.filter(
         schedule__meeting__schedule=F('schedule'),
-        session__in=[s.pk for m in meetings for s in m.sessions]
+        session__in=[s.pk for m in meetings for s in m.sessions],
+        timeslot__time__gte=today,
     ).order_by(
         'schedule__meeting__date', 'session__type', 'timeslot__time'
     ).select_related(

ietf/message/admin.py

@@ -5,7 +5,7 @@ from ietf.message.models import Message, MessageAttachment, SendQueue, Announcem
 class MessageAdmin(admin.ModelAdmin):
     list_display = ["subject", "by", "time", "groups"]
     search_fields = ["subject", "body"]
-    raw_id_fields = ["by"]
+    raw_id_fields = ["by", "related_groups", "related_docs"]
     ordering = ["-time"]
 
     def groups(self, instance):

ietf/name/fixtures/names.json

@@ -2292,6 +2292,19 @@
     "model": "doc.state",
     "pk": 156
   },
+  {
+    "fields": {
+      "desc": "This charter's group was replaced.",
+      "name": "Replaced",
+      "next_states": [],
+      "order": 0,
+      "slug": "replaced",
+      "type": "charter",
+      "used": true
+    },
+    "model": "doc.state",
+    "pk": 157
+  },
   {
     "fields": {
       "label": "State"
@@ -9681,7 +9694,7 @@
   {
     "fields": {
       "desc": "Formation of the group (most likely a BoF or Proposed WG) was abandoned",
-      "name": "Abandonded",
+      "name": "Abandoned",
       "order": 0,
       "used": true
     },
@@ -9750,7 +9763,7 @@
   },
   {
     "fields": {
-      "desc": "Replaced by dnssd",
+      "desc": "Replaced by a group with a different acronym",
       "name": "Replaced",
       "order": 0,
       "used": true
@@ -11448,6 +11461,16 @@
     "model": "name.roomresourcename",
     "pk": "u-shape"
   },
+  {
+    "fields": {
+      "desc": "WebEx support",
+      "name": "WebEx session",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.roomresourcename",
+    "pk": "webex"
+  },
   {
     "fields": {
       "desc": "",
@@ -14226,7 +14249,7 @@
     "fields": {
       "command": "xym",
       "switch": "--version",
-      "time": "2020-01-16T00:12:27.984",
+      "time": "2020-02-19T00:13:43.554",
       "used": true,
       "version": "xym 0.4"
     },
@@ -14237,7 +14260,7 @@
     "fields": {
       "command": "pyang",
       "switch": "--version",
-      "time": "2020-01-16T00:12:29.007",
+      "time": "2020-02-19T00:13:44.450",
       "used": true,
       "version": "pyang 2.1.1"
     },
@@ -14248,7 +14271,7 @@
     "fields": {
       "command": "yanglint",
       "switch": "--version",
-      "time": "2020-01-16T00:12:29.206",
+      "time": "2020-02-19T00:13:44.597",
       "used": true,
       "version": "yanglint 0.14.80"
     },
@@ -14259,9 +14282,9 @@
     "fields": {
       "command": "xml2rfc",
       "switch": "--version",
-      "time": "2020-01-16T00:12:30.657",
+      "time": "2020-02-19T00:13:45.481",
       "used": true,
-      "version": "xml2rfc 2.37.3"
+      "version": "xml2rfc 2.40.0"
     },
     "model": "utils.versioninfo",
     "pk": 4

ietf/nomcom/factories.py

@@ -11,7 +11,7 @@ from ietf.person.factories import PersonFactory, UserFactory
 
 import debug                            # pyflakes:ignore
 
-cert = '''-----BEGIN CERTIFICATE-----
+cert = b'''-----BEGIN CERTIFICATE-----
 MIIDHjCCAgagAwIBAgIJAKDCCjbQboJzMA0GCSqGSIb3DQEBCwUAMBMxETAPBgNV
 BAMMCE5vbUNvbTE1MB4XDTE0MDQwNDIxMTQxNFoXDTE2MDQwMzIxMTQxNFowEzER
 MA8GA1UEAwwITm9tQ29tMTUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
@@ -32,7 +32,7 @@ toX3j+FUe2UiUak3ACXdrOPSsFP0KRrFwuMnuHHXkGj/Uw==
 -----END CERTIFICATE-----
 '''
 
-key = '''-----BEGIN PRIVATE KEY-----
+key = b'''-----BEGIN PRIVATE KEY-----
 MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC2QXCsAitYSOgP
 Yor77zQnEeHuVqlcuhpH1wpKB+N6WcScA5N3AnX9uZEFOt6McJ+MCiHECdqDlH6n
 pQTJlpCpIVgAD4B6xzjRBRww8d3lClA/kKwsKzuX93RS0Uv30hAD6q9wjqK/m6vR
@@ -75,7 +75,7 @@ def nomcom_kwargs_for_year(year=None, *args, **kwargs):
     if 'group__acronym' not in kwargs:
         kwargs['group__acronym'] = 'nomcom%d'%year
     if 'group__name' not in kwargs:
-        kwargs['group__name'] = 'TEST VERSION of IAB/IESG Nominating Committee %d/%d'%(year,year+1)
+        kwargs['group__name'] = 'TEST VERSION of NomCom %d/%d'%(year,year+1)
     return kwargs
 
 

ietf/nomcom/tests.py

@@ -16,7 +16,7 @@ from django.conf import settings
 from django.core.files import File
 from django.contrib.auth.models import User
 from django.urls import reverse
-from django.utils.encoding import force_text
+from django.utils.encoding import force_text, force_str
 
 import debug                            # pyflakes:ignore
 
@@ -50,12 +50,12 @@ def get_cert_files():
         client_test_cert_files = generate_cert()
     return client_test_cert_files
 
-def build_test_public_keys_dir(obj):
+def setup_test_public_keys_dir(obj):
     obj.saved_nomcom_public_keys_dir = settings.NOMCOM_PUBLIC_KEYS_DIR
     obj.nomcom_public_keys_dir = obj.tempdir('nomcom-public-keys')
     settings.NOMCOM_PUBLIC_KEYS_DIR = obj.nomcom_public_keys_dir
 
-def clean_test_public_keys_dir(obj):
+def teardown_test_public_keys_dir(obj):
     settings.NOMCOM_PUBLIC_KEYS_DIR = obj.saved_nomcom_public_keys_dir
     shutil.rmtree(obj.nomcom_public_keys_dir)
 
@@ -68,7 +68,7 @@ class NomcomViewsTest(TestCase):
         return response
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         nomcom_test_data()
         self.cert_file, self.privatekey_file = get_cert_files()
         self.year = NOMCOM_YEAR
@@ -97,7 +97,7 @@ class NomcomViewsTest(TestCase):
         self.public_nominate_newperson_url = reverse('ietf.nomcom.views.public_nominate_newperson', kwargs={'year': self.year})
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def access_member_url(self, url):
         login_testing_unauthorized(self, COMMUNITY_USER, url)
@@ -941,12 +941,12 @@ class NomineePositionStateSaveTest(TestCase):
     """Tests for the NomineePosition save override method"""
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         nomcom_test_data()
         self.nominee = Nominee.objects.get(email__person__user__username=COMMUNITY_USER)
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_state_autoset(self):
         """Verify state is autoset correctly"""
@@ -976,13 +976,13 @@ class NomineePositionStateSaveTest(TestCase):
 class FeedbackTest(TestCase):
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
 
         nomcom_test_data()
         self.cert_file, self.privatekey_file = get_cert_files()
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_encrypted_comments(self):
 
@@ -1009,7 +1009,7 @@ class FeedbackTest(TestCase):
 class ReminderTest(TestCase):
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         nomcom_test_data()
         self.nomcom = get_nomcom_by_year(NOMCOM_YEAR)
         self.cert_file, self.privatekey_file = get_cert_files()
@@ -1051,7 +1051,7 @@ class ReminderTest(TestCase):
         feedback.nominees.add(n)
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_is_time_to_send(self):
         self.nomcom.reminder_interval = 4
@@ -1107,14 +1107,14 @@ class ReminderTest(TestCase):
 class InactiveNomcomTests(TestCase):
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory.create(**nomcom_kwargs_for_year(group__state_id='conclude'))
         self.plain_person = PersonFactory.create()
         self.chair = self.nc.group.role_set.filter(name='chair').first().person
         self.member = self.nc.group.role_set.filter(name='member').first().person
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_feedback_closed(self):
         for view in ['ietf.nomcom.views.public_feedback', 'ietf.nomcom.views.private_feedback']:
@@ -1301,7 +1301,7 @@ class InactiveNomcomTests(TestCase):
 class FeedbackLastSeenTests(TestCase):
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory.create(**nomcom_kwargs_for_year())
         self.author = PersonFactory.create().email_set.first().address
         self.member = self.nc.group.role_set.filter(name='member').first().person
@@ -1320,7 +1320,7 @@ class FeedbackLastSeenTests(TestCase):
         self.second_from_now = now + datetime.timedelta(seconds=1)
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_feedback_index_badges(self):
         url = reverse('ietf.nomcom.views.view_feedback',kwargs={'year':self.nc.year()})
@@ -1407,13 +1407,13 @@ class FeedbackLastSeenTests(TestCase):
 class NewActiveNomComTests(TestCase):
 
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory.create(**nomcom_kwargs_for_year())
         self.chair = self.nc.group.role_set.filter(name='chair').first().person
         self.saved_days_to_expire_nomination_link = settings.DAYS_TO_EXPIRE_NOMINATION_LINK
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
         settings.DAYS_TO_EXPIRE_NOMINATION_LINK = self.saved_days_to_expire_nomination_link
 
     def test_help(self):
@@ -1483,7 +1483,7 @@ class NewActiveNomComTests(TestCase):
         login_testing_unauthorized(self,self.chair.user.username,url)
         response = self.client.get(url)
         self.assertEqual(response.status_code,200)
-        response = self.client.post(url,{'key':key})
+        response = self.client.post(url,{'key': force_str(key)})
         self.assertEqual(response.status_code,302)
 
     def test_email_pasting(self):
@@ -1870,13 +1870,13 @@ class NoPublicKeyTests(TestCase):
         
 class AcceptingTests(TestCase):
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory(**nomcom_kwargs_for_year())
         self.plain_person = PersonFactory.create()
         self.member = self.nc.group.role_set.filter(name='member').first().person
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_public_accepting_nominations(self):
         url = reverse('ietf.nomcom.views.public_nominate',kwargs={'year':self.nc.year()})
@@ -1977,12 +1977,12 @@ class AcceptingTests(TestCase):
 
 class ShowNomineeTests(TestCase):
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory(**nomcom_kwargs_for_year())
         self.plain_person = PersonFactory.create()
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
 
     def test_feedback_pictures(self):
         url = reverse('ietf.nomcom.views.public_nominate',kwargs={'year':self.nc.year()})
@@ -1998,13 +1998,13 @@ class ShowNomineeTests(TestCase):
 
 class TopicTests(TestCase):
     def setUp(self):
-        build_test_public_keys_dir(self)
+        setup_test_public_keys_dir(self)
         self.nc = NomComFactory(**nomcom_kwargs_for_year(populate_topics=False))
         self.plain_person = PersonFactory.create()
         self.chair = self.nc.group.role_set.filter(name='chair').first().person
 
     def tearDown(self):
-        clean_test_public_keys_dir(self)
+        teardown_test_public_keys_dir(self)
     
     def testAddEditListRemoveTopic(self):
         self.assertFalse(self.nc.topic_set.exists())

ietf/nomcom/utils.py

@@ -166,7 +166,7 @@ def retrieve_nomcom_private_key(request, year):
 
     command = "%s bf -d -in /dev/stdin -k \"%s\" -a"
     code, out, error = pipe(command % (settings.OPENSSL_COMMAND,
-                                       settings.SECRET_KEY), private_key.encode('utf-8'))
+                                       settings.SECRET_KEY), private_key)
     if code != 0:
         log("openssl error: %s:\n  Error %s: %s" %(command, code, error))        
     return out
@@ -178,7 +178,7 @@ def store_nomcom_private_key(request, year, private_key):
     else:
         command = "%s bf -e -in /dev/stdin -k \"%s\" -a"
         code, out, error = pipe(command % (settings.OPENSSL_COMMAND,
-                                           settings.SECRET_KEY), private_key.encode('utf-8'))
+                                           settings.SECRET_KEY), private_key)
         if code != 0:
             log("openssl error: %s:\n  Error %s: %s" %(command, code, error))        
         if error:

ietf/nomcom/views.py

@@ -7,15 +7,16 @@ import re
 from collections import OrderedDict, Counter
 
 from django.conf import settings
+from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import AnonymousUser
-from django.contrib import messages
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.urls import reverse
+from django.forms.models import modelformset_factory, inlineformset_factory
 from django.http import Http404, HttpResponseRedirect, HttpResponseForbidden
 from django.shortcuts import render, get_object_or_404, redirect
 from django.template.loader import render_to_string
-from django.forms.models import modelformset_factory, inlineformset_factory
+from django.urls import reverse
+from django.utils.encoding import force_bytes
 
 
 from ietf.dbtemplate.models import DBTemplate
@@ -117,7 +118,7 @@ def private_key(request, year):
     if request.method == 'POST':
         form = PrivateKeyForm(data=request.POST)
         if form.is_valid():
-            store_nomcom_private_key(request, year, form.cleaned_data.get('key', ''))
+            store_nomcom_private_key(request, year, force_bytes(form.cleaned_data.get('key', '')))
             return HttpResponseRedirect(back_url)
     else:
         form = PrivateKeyForm()

ietf/person/templatetags/person_filters.py

@@ -22,4 +22,22 @@ def person_by_name(name):
         return None
     alias = Alias.objects.filter(name=name).first()
     return alias.person if alias else None
-    
+
+@register.inclusion_tag('person/person_link.html')
+def person_link(person, **kwargs):
+    title = kwargs.get('title', '')
+    cls = kwargs.get('class', '')
+    name = person.name
+    plain_name = person.plain_name()
+    email = person.email_address()
+    return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls}
+
+
+@register.inclusion_tag('person/person_link.html')
+def email_person_link(email, **kwargs):
+    title = kwargs.get('title', '')
+    cls = kwargs.get('class', '')
+    name = email.person.name
+    plain_name = email.person.plain_name()
+    email = email.address
+    return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls}

ietf/review/admin.py

@@ -69,7 +69,7 @@ class ReviewAssignmentAdmin(simple_history.admin.SimpleHistoryAdmin):
     list_display = ["review_request", "reviewer", "assigned_on", "result"]
     list_filter = ["result", "state"]
     ordering = ["-id"]
-    raw_id_fields = ["reviewer", "result", "review"]
+    raw_id_fields = ["review_request", "reviewer", "result", "review"]
     search_fields = ["review_request__doc__name"]
 
 admin.site.register(ReviewAssignment, ReviewAssignmentAdmin)

ietf/review/mailarch.py

@@ -5,14 +5,14 @@
 # various utilities for working with the mailarch mail archive at
 # mailarchive.ietf.org
 
+import base64
 import contextlib
 import datetime
-import tarfile
-import mailbox
-import tempfile
-import hashlib
-import base64
 import email.utils
+import hashlib
+import mailbox
+import tarfile
+import tempfile
 
 from urllib.parse import urlencode
 from urllib.request import urlopen
@@ -22,7 +22,7 @@ import debug                            # pyflakes:ignore
 from pyquery import PyQuery
 
 from django.conf import settings
-from django.utils.encoding import force_bytes
+from django.utils.encoding import force_bytes, force_str
 
 def list_name_from_email(list_email):
     if not list_email.endswith("@ietf.org"):
@@ -37,7 +37,7 @@ def hash_list_message_id(list_name, msgid):
     # and rightmost "=" signs are (optionally) stripped
     sha = hashlib.sha1(force_bytes(msgid))
     sha.update(force_bytes(list_name))
-    return base64.urlsafe_b64encode(sha.digest()).rstrip(b"=")
+    return force_str(base64.urlsafe_b64encode(sha.digest()).rstrip(b"="))
 
 def construct_query_urls(doc, team, query=None):
     list_name = list_name_from_email(team.list_email)
@@ -94,8 +94,8 @@ def retrieve_messages_from_mbox(mbox_fileobj):
                 "splitfrom": email.utils.parseaddr(msg["From"]),
                 "subject": msg["Subject"],
                 "content": content.replace("\r\n", "\n").replace("\r", "\n").strip("\n"),
-                "message_id": email.utils.unquote(msg["Message-ID"]),
+                "message_id": email.utils.unquote(msg["Message-ID"].strip()),
-                "url": email.utils.unquote(msg["Archived-At"]),
+                "url": email.utils.unquote(msg["Archived-At"].strip()),
                 "date": msg["Date"],
                 "utcdate": (utcdate.date().isoformat(), utcdate.time().isoformat()) if utcdate else ("", ""),
             })
@@ -106,6 +106,8 @@ def retrieve_messages(query_data_url):
     """Retrieve and return selected content from mailarch."""
     res = []
 
+    # This has not been rewritten to use requests.get() because get() does
+    # not handle file URLs out of the box, which we need for tesing
     with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj:
         content_type = fileobj.info()["Content-type"]
         if not content_type.startswith("application/x-tar"):

ietf/review/migrations/0023_historicalreviewersettings_change_reason_text_field.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Copyright The IETF Trust 2020, All Rights Reserved
+# Generated by Django 1.11.26 on 2019-12-21 11:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0022_reviewer_queue_policy_and_request_assignment_next'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='historicalreviewersettings',
+            name='history_change_reason',
+            field=models.TextField(null=True),
+        ),
+    ]

ietf/review/tests.py

@@ -10,15 +10,15 @@ class HashTest(TestCase):
 
     def test_hash_list_message_id(self):
         for list, msgid, hash in (
-                ('ietf', '156182196167.12901.11966487185176024571@ietfa.amsl.com',  b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
+                ('ietf', '156182196167.12901.11966487185176024571@ietfa.amsl.com',  'lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
-                ('codesprints', 'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',       b'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
+                ('codesprints', 'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',       'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
-                ('xml2rfc', '3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',  b'g6DN4SxJGDrlSuKsubwb6rRSePU'),
+                ('xml2rfc', '3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',  'g6DN4SxJGDrlSuKsubwb6rRSePU'),
-                (u'ietf', u'156182196167.12901.11966487185176024571@ietfa.amsl.com',b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
+                (u'ietf', u'156182196167.12901.11966487185176024571@ietfa.amsl.com','lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
-                (u'codesprints', u'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',     b'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
+                (u'codesprints', u'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',     'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
-                (u'xml2rfc', u'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',b'g6DN4SxJGDrlSuKsubwb6rRSePU'),
+                (u'xml2rfc', u'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org','g6DN4SxJGDrlSuKsubwb6rRSePU'),
-                (b'ietf', b'156182196167.12901.11966487185176024571@ietfa.amsl.com',b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
+                (b'ietf', b'156182196167.12901.11966487185176024571@ietfa.amsl.com','lr6RtZ4TiVMZn1fZbykhkXeKhEk'),
-                (b'codesprints', b'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',     b'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
+                (b'codesprints', b'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org',     'N1nFHHUXiFWYtdzBgjtqzzILFHI'),
-                (b'xml2rfc', b'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',b'g6DN4SxJGDrlSuKsubwb6rRSePU'),
+                (b'xml2rfc', b'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org','g6DN4SxJGDrlSuKsubwb6rRSePU'),
             ):
             self.assertEqual(hash, hash_list_message_id(list, msgid))
             

ietf/secr/groups/forms.py

@@ -74,6 +74,12 @@ class GroupModelForm(forms.ModelForm):
             if lsgc:
                 self.fields['liaison_contacts'].initial = lsgc.contacts
         
+    def clean_acronym(self):
+        acronym = self.cleaned_data['acronym']
+        if any(x.isupper() for x in acronym):
+            raise forms.ValidationError('Capital letters not allowed in group acronym')
+        return acronym
+
     def clean_parent(self):
         parent = self.cleaned_data['parent']
         type = self.cleaned_data['type']

ietf/secr/groups/tests.py

@@ -82,6 +82,22 @@ class GroupsTest(TestCase):
         response = self.client.post(url,post_data)
         self.assertEqual(response.status_code, 200)
 
+    def test_add_group_capital_acronym(self):
+        area = GroupFactory(type_id='area')
+        url = reverse('ietf.secr.groups.views.add')
+        post_data = {'acronym':'TEST',
+                     'name':'Test Group',
+                     'type':'wg',
+                     'status':'active',
+                     'parent':area.id,
+                     'awp-TOTAL_FORMS':'2',
+                     'awp-INITIAL_FORMS':'0',
+                     'submit':'Save'}
+        self.client.login(username="secretary", password="secretary+password")
+        response = self.client.post(url,post_data)
+        self.assertEqual(response.status_code, 200)
+        self.assertContains(response, 'Capital letters not allowed in group acronym')
+
     # ------- Test View -------- #
     def test_view(self):
         MeetingFactory(type_id='ietf')

ietf/secr/groups/urls.py

@@ -7,7 +7,6 @@ urlpatterns = [
     url(r'^$', views.search),
     url(r'^add/$', views.add),
     url(r'^blue-dot-report/$', views.blue_dot),
-    url(r'^search/$', views.search),
     #(r'^ajax/get_ads/$', views.get_ads),
     url(r'^%(acronym)s/$' % settings.URL_REGEXPS, views.view),
     url(r'^%(acronym)s/delete/(?P<id>\d{1,6})/$' % settings.URL_REGEXPS, views.delete_role),

ietf/secr/proceedings/reports.py

@@ -0,0 +1,103 @@
+import datetime
+
+from django.template.loader import render_to_string
+
+from ietf.meeting.models import Meeting
+from ietf.doc.models import DocEvent, Document
+from ietf.secr.proceedings.proc_utils import get_progress_stats
+
+def report_id_activity(start,end):
+
+    # get previous meeting
+    meeting = Meeting.objects.filter(date__lt=datetime.datetime.now(),type='ietf').order_by('-date')[0]
+    syear,smonth,sday = start.split('-')
+    eyear,emonth,eday = end.split('-')
+    sdate = datetime.datetime(int(syear),int(smonth),int(sday))
+    edate = datetime.datetime(int(eyear),int(emonth),int(eday))
+    
+    #queryset = Document.objects.filter(type='draft').annotate(start_date=Min('docevent__time'))
+    new_docs = Document.objects.filter(type='draft').filter(docevent__type='new_revision',
+                                                            docevent__newrevisiondocevent__rev='00',
+                                                            docevent__time__gte=sdate,
+                                                            docevent__time__lte=edate)
+    new = new_docs.count()
+    updated = 0
+    updated_more = 0
+    for d in new_docs:
+        updates = d.docevent_set.filter(type='new_revision',time__gte=sdate,time__lte=edate).count()
+        if updates > 1:
+            updated += 1
+        if updates > 2:
+            updated_more +=1
+    
+    # calculate total documents updated, not counting new, rev=00
+    result = set()
+    events = DocEvent.objects.filter(doc__type='draft',time__gte=sdate,time__lte=edate)
+    for e in events.filter(type='new_revision').exclude(newrevisiondocevent__rev='00'):
+        result.add(e.doc)
+    total_updated = len(result)
+    
+    # calculate sent last call
+    last_call = events.filter(type='sent_last_call').count()
+    
+    # calculate approved
+    approved = events.filter(type='iesg_approved').count()
+    
+    # get 4 weeks
+    monday = Meeting.get_current_meeting().get_ietf_monday()
+    cutoff = monday + datetime.timedelta(days=3)
+    ff1_date = cutoff - datetime.timedelta(days=28)
+    #ff2_date = cutoff - datetime.timedelta(days=21)
+    #ff3_date = cutoff - datetime.timedelta(days=14)
+    #ff4_date = cutoff - datetime.timedelta(days=7)
+    
+    ff_docs = Document.objects.filter(type='draft').filter(docevent__type='new_revision',
+                                                           docevent__newrevisiondocevent__rev='00',
+                                                           docevent__time__gte=ff1_date,
+                                                           docevent__time__lte=cutoff)
+    ff_new_count = ff_docs.count()
+    ff_new_percent = format(ff_new_count / float(new),'.0%')
+    
+    # calculate total documents updated in final four weeks, not counting new, rev=00
+    result = set()
+    events = DocEvent.objects.filter(doc__type='draft',time__gte=ff1_date,time__lte=cutoff)
+    for e in events.filter(type='new_revision').exclude(newrevisiondocevent__rev='00'):
+        result.add(e.doc)
+    ff_update_count = len(result)
+    ff_update_percent = format(ff_update_count / float(total_updated),'.0%')
+    
+    #aug_docs = augment_with_start_time(new_docs)
+    '''
+    ff1_new = aug_docs.filter(start_date__gte=ff1_date,start_date__lt=ff2_date)
+    ff2_new = aug_docs.filter(start_date__gte=ff2_date,start_date__lt=ff3_date)
+    ff3_new = aug_docs.filter(start_date__gte=ff3_date,start_date__lt=ff4_date)
+    ff4_new = aug_docs.filter(start_date__gte=ff4_date,start_date__lt=edate)
+    ff_new_iD = ff1_new + ff2_new + ff3_new + ff4_new
+    '''
+    context = {'meeting':meeting,
+               'new':new,
+               'updated':updated,
+               'updated_more':updated_more,
+               'total_updated':total_updated,
+               'last_call':last_call,
+               'approved':approved,
+               'ff_new_count':ff_new_count,
+               'ff_new_percent':ff_new_percent,
+               'ff_update_count':ff_update_count,
+               'ff_update_percent':ff_update_percent}
+    
+    report = render_to_string('proceedings/report_id_activity.txt', context)
+    
+    return report
+    
+def report_progress_report(start_date,end_date):
+    syear,smonth,sday = start_date.split('-')
+    eyear,emonth,eday = end_date.split('-')
+    sdate = datetime.datetime(int(syear),int(smonth),int(sday))
+    edate = datetime.datetime(int(eyear),int(emonth),int(eday))
+    
+    context = get_progress_stats(sdate,edate)
+    
+    report = render_to_string('proceedings/report_progress_report.txt', context)
+    
+    return report

ietf/secr/proceedings/tests_reports.py

@@ -0,0 +1,34 @@
+import datetime
+import debug    # pyflakes:ignore
+
+from ietf.doc.factories import DocumentFactory,NewRevisionDocEventFactory
+from ietf.secr.proceedings.reports import report_id_activity, report_progress_report
+from ietf.utils.test_utils import TestCase
+from ietf.meeting.factories import MeetingFactory
+
+class ReportsTestCase(TestCase):
+
+    def test_report_id_activity(self):
+
+        today = datetime.datetime.today()
+        yesterday = today - datetime.timedelta(days=1)
+        last_quarter = today - datetime.timedelta(days=3*30)
+        next_week = today+datetime.timedelta(days=7)
+
+        m1 = MeetingFactory(type_id='ietf',date=last_quarter)
+        m2 = MeetingFactory(type_id='ietf',date=next_week,number=int(m1.number)+1)
+
+        doc = DocumentFactory(type_id='draft',time=yesterday,rev="00")
+        NewRevisionDocEventFactory(doc=doc,time=today,rev="01")
+        result = report_id_activity(m1.date.strftime("%Y-%m-%d"),m2.date.strftime("%Y-%m-%d"))
+        self.assertTrue('IETF Activity since last IETF Meeting' in result)
+
+    def test_report_progress_report(self):
+        today = datetime.datetime.today()
+        last_quarter = today - datetime.timedelta(days=3*30)
+        next_week = today+datetime.timedelta(days=7)
+
+        m1 = MeetingFactory(type_id='ietf',date=last_quarter)
+        m2 = MeetingFactory(type_id='ietf',date=next_week,number=int(m1.number)+1)
+        result = report_progress_report(m1.date.strftime('%Y-%m-%d'),m2.date.strftime('%Y-%m-%d'))
+        self.assertTrue('IETF Activity since last IETF Meeting' in result)

ietf/secr/sreq/forms.py

@@ -96,7 +96,7 @@ class SessionForm(forms.Form):
         self.fields['length_session1'].widget.attrs['onClick'] = "if (check_num_session(1)) this.disabled=true;"
         self.fields['length_session2'].widget.attrs['onClick'] = "if (check_num_session(2)) this.disabled=true;"
         self.fields['length_session3'].widget.attrs['onClick'] = "if (check_third_session()) { this.disabled=true;}"
-        self.fields['comments'].widget = forms.Textarea(attrs={'rows':'6','cols':'65'})
+        self.fields['comments'].widget = forms.Textarea(attrs={'rows':'3','cols':'65'})
 
         group_acronym_choices = [('','--Select WG(s)')] + list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym','acronym').order_by('acronym'))
         for i in range(1, 4):

ietf/secr/sreq/views.py

@@ -592,10 +592,17 @@ def new(request, acronym):
     # the "previous" querystring causes the form to be returned
     # pre-populated with data from last meeeting's session request
     elif request.method == 'GET' and 'previous' in request.GET:
-        previous_meeting = Meeting.objects.get(number=str(int(meeting.number) - 1))
+        latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first()
-        previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id')
+        if latest_session:
-        if not previous_sessions:
+            previous_meeting = Meeting.objects.get(number=latest_session.meeting.number)
-            messages.warning(request, 'This group did not meet at %s' % previous_meeting)
+            previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id')
+            if not previous_sessions:
+                messages.warning(request, 'This group did not meet at %s' % previous_meeting)
+                return redirect('ietf.secr.sreq.views.new', acronym=acronym)
+            else:
+                messages.info(request, 'Fetched session info from %s' % previous_meeting)
+        else:
+            messages.warning(request, 'Did not find any previous meeting')
             return redirect('ietf.secr.sreq.views.new', acronym=acronym)
 
         initial = get_initial_session(previous_sessions, prune_conflicts=True)

ietf/secr/templates/drafts/abstract.html

@@ -1,31 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Abstract{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.canonical_name }}</a>
-  &raquo; Abstract
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>View Abstract - {{ draft.canonical_name }}</h2>
-    <p>
-      {{ draft.abstract }}
-    </p> 
-      <div class="button-group">
-        <ul>
-          <li><button onclick="window.location='../'">Back to View</button></li>
-        </ul>
-      </div> <!-- button-group -->
-
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/approvals.html

@@ -1,47 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Approvals{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; Drafts Approvals
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Approvals</h2>
-    <form id="drafts-approvals-form" enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table class="full-width">
-	{{ form.as_table }}
-      </table>
-    </form>
-
-  <div class="inline-group">
-    <h2>Approved Drafts</h2>
-<table id="drafts-approved" class="center" cellspacing="0">
-  <thead>
-    <tr> 
-      <th>Filename</th>
-      <th>Approved Date</th>
-      <th>Approved by</th>
-    </tr>
-  </thead>
-  <tbody>
-    {% for doc in approved %}
-       <tr class="{% cycle 'row1' 'row2' %}">
-         <td>{{ doc.name }}</a></td>
-         <td>{{ doc.time|date:"Y-m-d" }}</td>
-         <td>{{ doc.by }}</td>
-       </tr>
-    {% endfor %}
-  </tbody>
-</table>
-  </div> <!-- inline-group -->
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/authors.html

@@ -1,74 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Authors{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <link rel="stylesheet" type="text/css" href="{% static 'secr/css/jquery-ui-1.11.4.custom.css' %}" />
-  <script type="text/javascript" src="{% static 'secr/js/jquery-ui-1.11.4.custom.min.js' %}"></script>
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Authors
-{% endblock %}
-
-{% block content %}
-
-<div class="module">
-  <h2>Authors</h2>   
-    <table class="full-width">
-      <thead>
-        <tr>
-          <th>Name</th>
-          <th>Email</th>
-          <th>Affiliation</th>
-          <th>Country</th>
-          <th>Order</th>
-          <th>Action</th>
-        </tr>
-      </thead>
-      <tbody>
-        {% for author in draft.documentauthor_set.all %}
-          <tr class="{% cycle 'row1' 'row2' %}">
-            <td>{{ author.person }}</td>
-            <td>{{ author.email }}</td>
-            <td>{{ author.affiliation }}</td>
-            <td>{{ author.country.name }}</td>
-            <td>{{ author.order }}</td>
-            <td><a href="{% url 'ietf.secr.drafts.views.author_delete' id=draft.pk oid=author.id %}">Delete</a></td>
-          </tr>
-        {% endfor %}
-      </tbody>
-    </table>
-  
-    <div class="inline-related">
-      <!-- <hr><br> -->
-      <h3>Add Author</h3>
-      <form id="groups-people" action="" method="post">{% csrf_token %}
-        {{ form.non_field_errors }}
-        <table class="full-width">
-          <tr>
-            <td>{{ form.person.errors }}{{ form.person }}{% if form.person.help_text %}<br>{{ form.person.help_text }}{% endif %}</td>
-            <td>{{ form.email.errors }}{{ form.email }}{% if form.email.help_text %}<br>{{ form.email.help_text }}{% endif %}</td>
-          </tr>
-          <tr>
-            <td>{{ form.affiliation.errors }}{{ form.affiliation }}{% if form.affiliation.help_text %}<br>{{ form.affiliation.help_text }}{% endif %}</td>
-            <td>{{ form.country.errors }}{{ form.country }}{% if form.country.help_text %}<br>{{ form.country.help_text }}{% endif %}</td>
-            <td><input type="submit" name="submit" value="Add" /></td>
-          </tr>
-        </table>    
-    </div> <!-- inline-related -->
-
-    <div class="button-group">
-      <ul>
-        <li><button type="submit" name="submit" value="Done">Done</button></li>
-      </ul>
-    </div> <!-- button-group -->
-    
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/confirm.html

@@ -1,49 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Confirm{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Confirm
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Confirm</h2>
-    <form enctype="multipart/form-data" action="{% url "ietf.secr.drafts.views.do_action" id=draft.name %}" method="post">{% csrf_token %}
-      
-      <table>
-        <tr><th>Action Selected:</th><td>{{ action }}</td></tr>
-        <tr><th>Draft:</th><td>{{ draft.name }}</td></tr>
-        {% for detail in details %}
-          <tr><th>{{ detail.label }}</th><td>{{ detail.value }}</td></tr>
-        {% endfor %}
-      </table>
-      
-      {% if email %}
-        <div class="inline-group">
-          <h2>Scheduled Email</h2>
-          <table id="draft-confirm-email">
-            <tr><th>To:</th><td>{{ email.to }}</td></tr>
-            <tr><th>CC:</th><td>{{ email.cc }}</td></tr>
-            <tr><th>Subject:</th><td>{{ email.subject }}</td></tr>
-            <tr><th>Body:</th><td>{{ email.body }}</td></tr>
-          </table>
-        </div>
-      {% endif %}
-      
-      {{ form }}
-
-      {% include "includes/buttons_save_cancel.html" %}
-
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/dates.html

@@ -1,36 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Submission Dates{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; Submission Dates
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Submission Dates</h2>
-      <table class="full-width">
-        <tr><td>First Cut Off (Initial Version)</td><td>{{ meeting.get_first_cut_off }}</td></tr>
-        <tr><td>Second Cut Off (Update Version)</td><td>{{ meeting.get_second_cut_off }}</td></tr>
-        <tr><td>IETF Monday</td><td>{{ meeting.get_ietf_monday }}</td></tr>
-        <tr><td>All I-Ds will be processed by</td><td></td></tr>
-        <tr><td>Monday after IETF</td><td></td></tr>
-        <tr><td>Date for list of approved V-00 submissions from WG Chairs</td><td></td></tr>
-      </table>
-      
-      <div class="button-group">
-      <ul>
-        <li><button onclick="history.go(-1);return true">Back</button></li>
-      </ul>
-      </div> <!-- button-group -->
-  
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/edit.html

@@ -1,61 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Edit{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <link rel="stylesheet" type="text/css" href="{% static 'secr/css/jquery-ui-1.11.4.custom.css' %}" />
-  <script type="text/javascript" src="{% static 'secr/js/jquery-ui-1.11.4.custom.min.js' %}"></script>
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-
-  <link rel="stylesheet" href="{% static 'select2/select2.css' %}">
-  <link rel="stylesheet" href="{% static 'select2-bootstrap-css/select2-bootstrap.min.css' %}">
-  <script src="{% static 'select2/select2.min.js' %}"></script>
-  <script src="{% static 'ietf/js/select2-field.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Edit
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Edit</h2>
-    {{ form.non_field_errors }}
-    <form enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table id="draft-edit-table" class="full-width amstable">
-        <tr><th>Document Name:</th><td>{{ form.title.errors }}{{ form.title }}</td></tr>
-        <tr><th>Group:</th><td>{{ form.group.errors }}{{ form.group }}</td></tr>
-        <tr><th>Area Director:</th><td>{{ form.ad.errors }}{{ form.ad }}</td></tr>
-        <tr><th>Shepherd:</th><td>{{ form.shepherd.errors }}{{ form.shepherd }}</td></tr>
-        <tr><th>Notify:</th><td>{{ form.notify.errors }}{{ form.notify }}</td></tr>
-        <tr><th>State:</th><td>{{ form.state.errors }}{{ form.state }}</td></tr>
-        <tr><th>IESG State:</th><td>{{ form.iesg_state.errors }}{{ form.iesg_state }}</td></tr>
-        <tr><th>Stream:</th><td>{{ form.stream.errors }}{{ form.stream }}</td></tr>
-        <tr><th>Under Review by RFC Editor:</th><td>{{ form.review_by_rfc_editor.errors }}{{ form.review_by_rfc_editor }}</td></tr>
-        <tr><th>File Name:</th><td>{{ form.name.errors }}{{ form.name }} - {{ form.rev.errors }}{{ form.rev }}</td></tr>
-        <tr><th>Number of Pages:</th><td>{{ form.pages.errors }}{{ form.pages }}</td></tr>
-        <tr><th>Abstract:</th><td>{{ form.abstract.errors }}{{ form.abstract }}</td></tr>
-        <tr><th>Expiration Date:</th><td>{{ draft.expires|date:"M. d, Y" }}</td></tr>
-        <tr><th>Intended Std Level:</th><td>{{ form.intended_std_level.errors }}{{ form.intended_std_level }}</td></tr>
-        <tr><th>Standardization Level:</th><td>{{ form.std_level.errors }}{{ form.std_level }}</td></tr>
-        <tr><th>RFC Number:</th><td>{{ draft.rfc_number }}</td></tr>
-        <tr><th>Comments:</th><td>{{ form.internal_comments.errors }}{{ form.internal_comments }}</td></tr>
-        <tr><th>Replaced by:</th><td>{{ form.replaced_by.errors }}{{ form.replaced_by }}</td></tr>
-        <tr><th>Last Modified Date:</th><td>{{ draft.time }}</td></tr>
-      </table>
-
-  <div class="button-group">
-    <ul>
-      <li><button type="submit" name="submit" value="Save">Save</button></li>
-      <li><button type="submit" name="submit" value="Cancel">Cancel</button></li>
-    </ul>
-  </div> <!-- button-group -->
-
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/email.html

@@ -1,30 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Email{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Email
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Email</h2>
-    <form enctype="multipart/form-data" action="{% url "ietf.secr.drafts.views.confirm" id=draft.name %}" method="post">{% csrf_token %}
-      <table id="draft-email-table" class="full-width">
-	{{ form.as_table }}
-      </table>
-
-      {% include "includes/buttons_save_cancel.html" %}
-
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/error.html

@@ -1,23 +0,0 @@
-{% extends "base_site.html" %}
-
-{% block title %}Drafts{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-{% endblock %}
-
-{% block content %}
-
-<div class="module group-container">
-    <h2>Drafts - Error Message</h2>
-      An ERROR has occured:
-      <h3>{{ error }}</h3>
-      <div class="button-group">
-	<ul>
-	  <li><button onClick="window.location='../../'">Continue</button></li>
-	</ul>
-      </div> <!-- button-group -->
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/extend.html

@@ -1,32 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Extend{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Extend
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Extend Expiry</h2>
-    
-    <form enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table class="full-width">
-        <tr><th><label>Current Expiration Date:</label></th><td>{{ draft.expires|date:"Y-m-d" }}</td></tr>
-        {{ form.as_table }}
-      </table>
-
-      {% include "includes/buttons_save_cancel.html" %}
-
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/makerfc.html

@@ -1,81 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Make RFC{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Make RFC
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-  <div id="draft-view-col1">
-    <h2>Draft - Make RFC</h2>
-    <form enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table id="draft-makerfc-table" class="full-width">
-        {% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %}
-        <tr><th><label for="id_title">RFC Name</label></th><td colspan="3">{{ form.title.errors }}{{ form.title }}</td></tr>
-        <tr>
-          <th><label for="id_rfc_number">RFC Number</label></th><td>{{ form.rfc_number.errors }}{{ form.rfc_number }}</td>
-          <th><label for="id_txt_page_count">No. of Pages</label></th><td>{{ form.pages.errors }}{{ form.pages }}</td>
-        </tr>
-        <tr><th><label for="id_rfc_published_date">RFC Published Date</label></th><td>{{ form.rfc_published_date.errors }}{{ form.rfc_published_date }}</td></tr>
-        <tr>
-          <th><label for="id_status">RFC Std Level</label></th><td>{{ form.std_level.errors }}{{ form.std_level }}</td>
-          <th><label for="id_group_acronym">Group</label></th><td>{{ form.group.errors }}{{ form.group }}</td>
-        </tr>
-        {% comment %}
-        <tr>
-          <th><label for="id_proposed_date">Proposed Date</label></th><td>{{ form.proposed_date.errors }}{{ form.proposed_date }}</td>
-          <th><label for="id_draft_date">Draft Date</label></th><td>{{ form.draft_date.errors }}{{ form.draft_date }}</td>
-        </tr>
-        <tr>
-          <th><label for="id_standard_date">Standard Date</label></th><td>{{ form.standard_date.errors }}{{ form.standard_date }}</td>
-          <th><label for="id_historic_date">Historic Date</label></th><td>{{ form.historic_date.errors }}{{ form.historic_date }}</td>
-        </tr>
-        <tr>
-          <th><label for="id_fyi_number">FYI Number</label></th><td>{{ form.fyi_number.errors }}{{ form.fyi_number }}</td>
-          <th><label for="id_std_number">STD Number</label></th><td>{{ form.std_number.errors }}{{ form.std_number }}</td>
-        </tr>
-        {% endcomment %}
-        <tr><th><label for="id_comments">Comments</label></th><td colspan="3">{{ form.internal_comments.errors }}{{ form.internal_comments }}</td></tr>
-      </table>
-  </div> <!-- draft-view-col1 -->
-  <div id="draft-view-col2">
-    <div class="inline-related">
-      <h2>Author(s)</h2>
-        <table>
-        {% for author in draft.documentauthor_set.all %}
-          <tr><td><a href="{% url "ietf.secr.rolodex.views.view" id=author.person.pk %}">{{ author.person.name }}</a></td></tr>
-        {% endfor %}
-        </table>
-    </div> <!-- inline-related -->
-    <div class="inline-related add-form action-group">
-        <h2>Obsolete and Update Relationships</h2>
-          {{ obs_formset.management_form }}
-          {{ obs_formset.non_form_errors }}
-          <table id="draft-obsoletes-table" class="full-width">
-            {% for form in obs_formset.forms %}
-              
-              {% comment %}<tr><td colspan="2">{{ form.relation.errors }}{{ form.rfc.errors }}</td></tr>{% endcomment %}
-              <tr>
-                <td>{% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %}
-                {{ form.relation.errors }}{{ form.relation }}</td><td>{{ form.rfc.errors }}{{ form.rfc }}</td>
-              </tr>
-            {% endfor %}
-          </table>
-    </div> <!-- inline-related -->
-  </div> <!-- draft-view-col2 -->
-  
-   {% include "includes/buttons_save_cancel.html" %}
-  </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/message_extend.txt

@@ -1,8 +0,0 @@
-As you requested, the expiration date for
-{{ doc }} has been extended.  The draft
-will expire on {{ expire_date }} unless it is replaced by an updated version, or the 
-Secretariat has been notified that the document is under official review by the
-IESG or has been passed to the IRSG or RFC Editor for review and/or publication
-as an RFC.
-
-IETF Secretariat.

ietf/secr/templates/drafts/message_resurrect.txt

@@ -1,6 +0,0 @@
-As you requested, {{ doc }} has been resurrected.  The draft will expire on
-{{ expire_date }} unless it is replaced by an updated version, or the Secretariat has been notified that the
-document is under official review by the IESG or has been passed to the IRSG or RFC Editor for review and/or
-publication as an RFC.
-
-IETF Secretariat.

ietf/secr/templates/drafts/message_withdraw.txt

@@ -1,4 +0,0 @@
-As you requested, {{ doc }} 
-has been marked as withdrawn by the {{ by }} in the IETF Internet-Drafts database.
-
-IETF Secretariat.

ietf/secr/templates/drafts/report_nudge.html

@@ -1,26 +0,0 @@
-{% extends "base_site.html" %}
-{% load ams_filters %}
-
-{% block title %}Drafts{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../../">Drafts</a>
-{% endblock %}
-
-{% block content %}
-
-<div class="module">
-    <h2>Nudge Report</h2>
-    <table>
-      {% for doc in docs %}
-      <tr>
-        <td><a href="{{ doc.get_absolute_url }}">{{ doc.name }}</a></td>
-        <td>{{ doc|get_published_date|date:"Y-m-d" }}</td>
-        <td></td>
-        <td>{% if doc.ad_name %}{{ doc.ad_name }}{% else %}&nbsp;{% endif %}</td>
-      </tr>
-      {% endfor %}
-    </table>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/search.html

@@ -1,34 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Search{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; Drafts 
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Search</h2>
-    <form id="draft-search-form" enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table id="draft-search-table" class="full-width amstable">
-        {{ form.as_table }}
-      </table>
-
-      {% include "includes/buttons_search.html" %}
-
-    </form>
-
-  <div class="inline-group">
-    <h2>Search Results</h2>
-      {% include "includes/draft_search_results.html" %}
-      {% if not_found %}{{ not_found }}{% endif %}
-  </div> <!-- inline-group -->
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/view.html

@@ -1,102 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - View{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }}
-  &raquo; <a href="../">Drafts</a>
-  &raquo; {{ draft.name }}
-{% endblock %}
-
-{% block content %}
-
-{% comment %}{{ request.user }}{% endcomment %}
-
-<div class="module draft-container">
-  <div id="draft-view-col1">
-    <h2>Draft - View</h2>
-      <table class="full-width">
-        <tr><td>Document Name:</td><td>{{ draft.title }}</td></tr>
-        <tr><td>Area:</td><td>{% if draft.group.parent %}<a href="{% url "ietf.secr.areas.views.view" name=draft.group.parent.acronym %}">{{ draft.group.parent }}{% endif %}</td></tr>
-        <tr><td>Group:</td><td>{% if draft.group %}<a href="{% url 'ietf.secr.groups.views.view' acronym=draft.group.acronym %}">{{ draft.group.acronym }}{% endif %}</td></tr>
-        <tr><td>Area Director:</td><td>{{ draft.ad }}</td></tr>
-        <tr><td>Shepherd:</td><td>{% if draft.shepherd %}{{ draft.shepherd.person }} &lt;{{ draft.shepherd.address }}&gt;{% else %}None{% endif %}</td></tr>
-        <tr><td>Notify:</td><td>{{ draft.notify }}</td></tr>
-        <tr><td>Document State:</td><td>{{ draft.get_state }}</td></tr>
-        <tr><td>IESG State:</td><td>{{ draft.iesg_state }}</td></tr>
-        <tr><td>Stream:</td><td>{{ draft.stream }}</td></tr>
-        <tr><td>Under Review by RFC Editor:</td><td>{% if draft.review_by_rfc_editor %}YES{% else %}NO{% endif %}</td></tr>
-        <tr><td>File Name:</td>
-          <td>{% if draft.expired_tombstone %}
-                {{ draft.filename }}
-                <div class=alert>This is a last active version - the tombstone was expired.</div>
-              {% else %}<a href="{% if is_development %}http://devr.amsl.com/ftp/{% else %}https://www.ietf.org/internet-drafts/{% endif %}{{ draft.name }}-{{ draft.rev }}.txt">{{ draft.name }}</a>
-              {% endif %}
-          </td></tr>
-        <tr><td>Document Aliases:</td>
-            <td>{% for alias in draft.docalias_set.all %}
-                  {% if not forloop.first %}, {% endif %}
-                  {{ alias.name }}
-                {% endfor %}
-            </td>
-        </tr>
-        <tr><td>Revision:</td><td>{{ draft.rev }}</td></tr>
-        <tr><td>Revision Date:</td><td>{{ draft.revision_date }}</td></tr>
-        <tr><td>Start Date:</td><td>{{ draft.start_date }}</td></tr>
-        <tr><td>Number of Pages:</td><td>{{ draft.pages }}</td></tr>
-        {% comment %}<tr><td>Local Path:</td><td>/ftp/internet-drafts/{{ draft.local_path|default_if_none:"" }}</td></tr>{% endcomment %}
-        <tr><td>Abstract:</td><td><a href="{% url 'ietf.secr.drafts.views.abstract' id=draft.name %}">Click here to view the abstract</td></tr>
-        <tr><td>Expiration Date:</td><td>{{ draft.expires|date:"M. d, Y" }}</td></tr>
-        <tr><td>Intended Status:</td><td>{{ draft.intended_std_level|default_if_none:"" }}</td></tr>
-        <tr><td>Standardization Level:</td><td>{{ draft.std_level|default_if_none:"" }}</td></tr>
-        <tr><td>RFC Number:</td><td>{{ draft.rfc_number|default_if_none:"" }}</td></tr>
-        <tr><td>Comments:</td><td>{{ draft.internal_comments|default_if_none:"" }}</td></tr>
-        <tr><td>Last Modified Date:</td><td>{{ draft.time }}</td></tr>
-        <tr><td>Replaced by:</td><td>{% if draft.replaced_by %}<a href="{% url "ietf.secr.drafts.views.view" id=draft.replaced_by.name %}">{{ draft.replaced_by.name }}{% endif %}</td></tr>
-        <tr><td>Related Documents:</td><td>{% for item in draft.relateddocument_set.all %}{% if not forloop.first %}, {% endif %}{{ item.relationship }} <a href="{% url "ietf.secr.drafts.views.view" id=item.target.document.pk %}">{{ item.target.name }}</a>{% endfor %}</td></tr>
-        <tr><td>Tags:</td>
-            <td>{% for tag in draft.tags.all %}
-                  {% if not forloop.first %}, {% endif %}
-                  {{ tag }}
-                {% endfor %}
-            </td>
-        </tr>
-      </table>
-
-  </div> <!-- draft-view-col1 -->
-  <div id="draft-view-col2">
-      <div class="inline-related">
-        <h2>Author(s)</h2>
-        <table class="full-width">
-        {% for author in draft.documentauthor_set.all %}
-          <tr><td><a href="{% url 'ietf.secr.rolodex.views.view' id=author.person.id %}">{{ author.person.name }}</a></td></tr>
-        {% endfor %}
-        </table>
-      </div> <!-- inline-related -->
-      <div class="inline-related action-group">
-        <h2>Actions</h2>
-          <ul>
-            <li><button {% if is_expired or is_withdrawn %}{% else %}disabled="disabled"{% endif %}onclick="window.location='{% url "ietf.secr.drafts.views.email" id=draft.name %}?action=resurrect'">Resurrect</button></li>
-            <li><button {% if is_active %}{% else %}disabled="disabled"{% endif %}onclick="window.location='extend/'">Extend Expiry</button></li>
-            <li><button {% if is_active %}{% else %}disabled="disabled"{% endif %}onclick="window.location='withdraw/'">Withdraw</button></li>
-          </ul>
-      </div> <!-- inline-related -->
-  </div> <!-- draft-view-col2 -->
-
-      <div class="button-group">
-	<ul>
-          <li><button onclick="window.location='edit/'">Edit</button></li>
-          <li><button onclick="window.location='authors/'">Authors</button></li>
-          {% comment %}
-          <li><button onclick="window.location='{% url "sec.ids.views.search" id=group.group_acronym.acronym_id %}'">Drafts</button></li>
-          <li><button onclick="window.location='{% url "sec.rfcs.views.search" id=group.group_acronym.acronym_id %}'">RFCs</button></li>
-          {% endcomment %}
-	</ul>
-      </div> <!-- button-group -->
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/drafts/withdraw.html

@@ -1,30 +0,0 @@
-{% extends "base_site.html" %}
-{% load staticfiles %}
-
-{% block title %}Drafts - Withdraw{% endblock %}
-
-{% block extrahead %}{{ block.super }}
-  <script type="text/javascript" src="{% static 'secr/js/utils.js' %}"></script>
-{% endblock %}
-
-{% block breadcrumbs %}{{ block.super }} 
-  &raquo; <a href="../">Drafts</a>
-  &raquo; <a href="../">{{ draft.name }}</a>
-  &raquo; Withdraw
-{% endblock %}
-
-{% block content %}
-
-<div class="module draft-container">
-    <h2>Draft - Withdraw</h2>
-    <form enctype="multipart/form-data" action="" method="post">{% csrf_token %}
-      <table class="full-width">
-	{{ form.as_table }}
-      </table>
-
-      {% include "includes/buttons_save_cancel.html" %}
-
-    </form>
-</div> <!-- module -->
-
-{% endblock %}

ietf/secr/templates/includes/draft_search_results.html

@@ -1,20 +0,0 @@
-<table id="draft-search-results" class="center" cellspacing="0">
-  <thead>
-    <tr> 
-      <th>Name</th>
-      <th>Group</th>
-      <th>Status</th>
-      <th>Intended Status</th>
-    </tr>
-  </thead>
-  <tbody>
-    {% for item in results %}
-       <tr class="{% cycle 'row1' 'row2' %}">
-         <td><a href="{% url "ietf.secr.drafts.views.view" id=item.name %}">{{ item.name }}</a></td>
-         <td>{{item.group.acronym}}</td>
-         <td>{{item.get_state}}</td>
-         <td>{{item.intended_std_level}}</td>
-       </tr>
-    {% endfor %}
-  </tbody>
-</table>

ietf/secr/templates/includes/sessions_request_form.html

@@ -60,7 +60,7 @@
             </td>
            </tr>
            <tr class="bg1">
-             <td valign="top">Special Requests:<br />&nbsp;<br />i.e. restrictions on meeting times / days, etc.</td>
+		   <td valign="top">Special Requests:<br />&nbsp;<br />i.e. restrictions on meeting times / days, etc.<br /> (limit 200 characters)</td>
              <td>{{ form.comments.errors }}{{ form.comments }}</td>
            </tr>
          </table>

ietf/secr/templates/main.html

@@ -16,7 +16,6 @@
    </td>
    <td>
      <h3>IDs and WGs Process</h3>
-       <li> <a href="{% url "ietf.secr.drafts.views.search" %}"><b>Drafts</b></a></li><br>
        <li> <a href="{% url "ietf.secr.areas.views.list_areas" %}"><b>Areas</b></a></li><br>
        <li> <a href="{% url 'ietf.secr.groups.views.search' %}"><b>Groups</b></a></li><br>
        <li> <a href="{% url 'ietf.secr.rolodex.views.search' %}"><b>Rolodex</b></a></li><br>
@@ -26,7 +25,6 @@
   <tr valign="top">
     <td>
       <h3>Meetings and Proceedings</h3>
-       <li> <a href="{% url 'ietf.secr.drafts.views.dates' %}"><b>Draft Submission Dates</b></a></li><br>
        <li> <a href="{% url "ietf.secr.sreq.views.main" %}"><b>Session Requests</b></a></li><br>
        <li> <a href="{% url 'ietf.secr.proceedings.views.main' %}"><b>Meeting Materials Manager (Proceedings)</b></a></li><br>
        <li> <a href="{% url "ietf.secr.meetings.views.main" %}"><b>Meeting Manager</b></a></li><br>

ietf/secr/urls.py

@@ -6,7 +6,6 @@ urlpatterns = [
     url(r'^announcement/', include('ietf.secr.announcement.urls')),
     url(r'^areas/', include('ietf.secr.areas.urls')),
     url(r'^console/', include('ietf.secr.console.urls')),
-    url(r'^drafts/', include('ietf.secr.drafts.urls')),
     url(r'^groups/', include('ietf.secr.groups.urls')),
     url(r'^meetings/', include('ietf.secr.meetings.urls')),
     url(r'^proceedings/', include('ietf.secr.proceedings.urls')),

ietf/settings.py

@@ -436,7 +436,6 @@ INSTALLED_APPS = [
     # IETF Secretariat apps
     'ietf.secr.announcement',
     'ietf.secr.areas',
-    'ietf.secr.drafts',
     'ietf.secr.groups',
     'ietf.secr.meetings',
     'ietf.secr.proceedings',
@@ -760,6 +759,9 @@ IDSUBMIT_ANNOUNCE_LIST_EMAIL = 'i-d-announce@ietf.org'
 
 # Interim Meeting Tool settings
 INTERIM_ANNOUNCE_FROM_EMAIL = 'IESG Secretary <iesg-secretary@ietf.org>'
+VIRTUAL_INTERIMS_REQUIRE_APPROVAL = True
+INTERIM_SESSION_MINIMUM_MINUTES = 30
+INTERIM_SESSION_MAXIMUM_MINUTES = 300
 
 # Days from meeting to day of cut off dates on submit -- cutoff_time_utc is added to this
 IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_00 = 13

ietf/submit/forms.py

@@ -84,6 +84,11 @@ class SubmissionBaseUploadForm(forms.Form):
         cutoff_00_str = cutoff_00.strftime("%Y-%m-%d %H:%M %Z")
         cutoff_01_str = cutoff_01.strftime("%Y-%m-%d %H:%M %Z")
         reopen_str    = reopen.strftime("%Y-%m-%d %H:%M %Z")
+
+        # Workaround for IETF107. This would be better handled by a refactor that allowed meetings to have no cutoff period.
+        if cutoff_01 >= reopen:
+            return
+
         if cutoff_00 == cutoff_01:
             if now.date() >= (cutoff_00.date() - meeting.idsubmit_cutoff_warning_days) and now.date() < cutoff_00.date():
                 self.cutoff_warning = ( 'The last submission time for Internet-Drafts before %s is %s.<br/><br/>' % (meeting, cutoff_00_str))
@@ -308,22 +313,26 @@ class SubmissionBaseUploadForm(forms.Form):
             txt_file.seek(0)
             try:
                 text = bytes.decode(self.file_info['txt'].charset)
+            #
+                self.parsed_draft = Draft(text, txt_file.name)
+                if self.filename == None:
+                    self.filename = self.parsed_draft.filename
+                elif self.filename != self.parsed_draft.filename:
+                    self.add_error('txt', "Inconsistent name information: xml:%s, txt:%s" % (self.filename, self.parsed_draft.filename))
+                if self.revision == None:
+                    self.revision = self.parsed_draft.revision
+                elif self.revision != self.parsed_draft.revision:
+                    self.add_error('txt', "Inconsistent revision information: xml:%s, txt:%s" % (self.revision, self.parsed_draft.revision))
+                if self.title == None:
+                    self.title = self.parsed_draft.get_title()
+                elif self.title != self.parsed_draft.get_title():
+                    self.add_error('txt', "Inconsistent title information: xml:%s, txt:%s" % (self.title, self.parsed_draft.get_title()))
             except (UnicodeDecodeError, LookupError) as e:
                 self.add_error('txt', 'Failed decoding the uploaded file: "%s"' % str(e))
-            #
+
-            self.parsed_draft = Draft(text, txt_file.name)
+        rev_error = validate_submission_rev(self.filename, self.revision)
-            if self.filename == None:
+        if rev_error:
-                self.filename = self.parsed_draft.filename
+            raise forms.ValidationError(rev_error)
-            elif self.filename != self.parsed_draft.filename:
-                self.add_error('txt', "Inconsistent name information: xml:%s, txt:%s" % (self.filename, self.parsed_draft.filename))
-            if self.revision == None:
-                self.revision = self.parsed_draft.revision
-            elif self.revision != self.parsed_draft.revision:
-                self.add_error('txt', "Inconsistent revision information: xml:%s, txt:%s" % (self.revision, self.parsed_draft.revision))
-            if self.title == None:
-                self.title = self.parsed_draft.get_title()
-            elif self.title != self.parsed_draft.get_title():
-                self.add_error('txt', "Inconsistent title information: xml:%s, txt:%s" % (self.title, self.parsed_draft.get_title()))
 
         # The following errors are likely noise if we have previous field
         # errors:

ietf/submit/tests.py

@@ -32,7 +32,7 @@ from ietf.meeting.factories import MeetingFactory
 from ietf.message.models import Message
 from ietf.name.models import FormalLanguageName
 from ietf.person.models import Person
-from ietf.person.factories import UserFactory, PersonFactory
+from ietf.person.factories import UserFactory, PersonFactory, EmailFactory
 from ietf.submit.models import Submission, Preapproval
 from ietf.submit.mail import add_submission_email, process_response_email
 from ietf.utils.mail import outbox, empty_outbox, get_payload
@@ -1005,6 +1005,40 @@ class SubmitTests(TestCase):
         q = PyQuery(r.content)
         self.assertEqual(len(q('input[type=file][name=txt]')), 1)
 
+    def test_no_blackout_at_all(self):
+        url = urlreverse('ietf.submit.views.upload_submission')
+
+        meeting = Meeting.get_current_meeting()
+        meeting.date = datetime.date.today()+datetime.timedelta(days=7)
+        meeting.save()
+        meeting.importantdate_set.filter(name_id='idcutoff').delete()
+        meeting.importantdate_set.create(name_id='idcutoff', date=datetime.date.today()+datetime.timedelta(days=7))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('input[type=file][name=txt]')), 1)        
+
+        meeting = Meeting.get_current_meeting()
+        meeting.date = datetime.date.today()
+        meeting.save()
+        meeting.importantdate_set.filter(name_id='idcutoff').delete()
+        meeting.importantdate_set.create(name_id='idcutoff', date=datetime.date.today())
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('input[type=file][name=txt]')), 1)        
+
+        meeting = Meeting.get_current_meeting()
+        meeting.date = datetime.date.today()-datetime.timedelta(days=1)
+        meeting.save()
+        meeting.importantdate_set.filter(name_id='idcutoff').delete()
+        meeting.importantdate_set.create(name_id='idcutoff', date=datetime.date.today()-datetime.timedelta(days=1))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code,200)
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('input[type=file][name=txt]')), 1)        
+
+        
     def submit_bad_file(self, name, formats):
         rev = ""
         group = None
@@ -1083,6 +1117,38 @@ class SubmitTests(TestCase):
         self.assertIn('Expected the PS file to have extension ".ps"', m)
         self.assertIn('Expected an PS file of type "application/postscript"', m)
 
+    def test_submit_file_in_archive(self):
+        name = "draft-authorname-testing-file-exists"
+        rev = '00'
+        formats = ['txt', 'xml']
+        group = None
+
+        # break early in case of missing configuration
+        self.assertTrue(os.path.exists(settings.IDSUBMIT_IDNITS_BINARY))
+
+        # get
+        url = urlreverse('ietf.submit.views.upload_submission')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        q = PyQuery(r.content)
+
+        # submit
+        for dir in [self.repository_dir, self.archive_dir, ]:
+            files = {}
+            for format in formats:
+                fn = os.path.join(dir, "%s-%s.%s" % (name, rev, format))
+                with io.open(fn, 'w') as f:
+                    f.write("a" * 2000)
+                files[format], author = submission_file(name, rev, group, format, "test_submission.%s" % format)
+
+            r = self.client.post(url, files)
+
+            self.assertEqual(r.status_code, 200)
+            q = PyQuery(r.content)
+            m = q('div.alert-danger').text()
+
+            self.assertIn('Unexpected files already in the archive', m)
+
     def test_submit_nonascii_name(self):
         name = "draft-authorname-testing-nonascii"
         rev = "00"
@@ -1770,6 +1836,26 @@ class ApiSubmitTests(TestCase):
         expected = "Upload of %s OK, confirmation requests sent to:\n  %s" % (name, author.formatted_email().replace('\n',''))
         self.assertContains(r, expected, status_code=200)
 
+    def test_api_submit_secondary_email_active(self):
+        person = PersonFactory()
+        email = EmailFactory(person=person)
+        r, author, name = self.post_submission('00', author=person, email=email.address)
+        for expected in [
+                "Upload of %s OK, confirmation requests sent to:" % (name, ),
+                author.formatted_email().replace('\n',''),
+            ]:
+            self.assertContains(r, expected, status_code=200)
+
+    def test_api_submit_secondary_email_inactive(self):
+        person = PersonFactory()
+        prim = person.email()
+        prim.primary = True
+        prim.save()
+        email = EmailFactory(person=person, active=False)
+        r, author, name = self.post_submission('00', author=person, email=email.address)
+        expected = "No such user: %s" % email.address
+        self.assertContains(r, expected, status_code=400)
+
     def test_api_submit_no_user(self):
         email='nonexistant.user@example.org'
         r, author, name = self.post_submission('00', email=email)

ietf/submit/utils.py

@@ -5,6 +5,7 @@
 import datetime
 import io
 import os
+import pathlib
 import re
 
 from typing import Callable, Optional # pyflakes:ignore
@@ -139,6 +140,15 @@ def validate_submission_rev(name, rev):
         if rev != expected:
             return 'Invalid revision (revision %02d is expected)' % expected
 
+        for dirname in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR, ]:
+            dir = pathlib.Path(dirname)
+            pattern = '%s-%02d.*' % (name, rev)
+            existing = list(dir.glob(pattern))
+            if existing:
+                plural = '' if len(existing) == 1 else 's'
+                files  = ', '.join([ f.name for f in existing ])
+                return 'Unexpected file%s already in the archive: %s' % (plural, files)
+
     replaced_by=has_been_replaced_by(name)
     if replaced_by:
         return 'This document has been replaced by %s' % ",".join(rd.name for rd in replaced_by)

ietf/submit/views.py

@@ -27,7 +27,7 @@ from ietf.group.utils import group_features_group_filter
 from ietf.ietfauth.utils import has_role, role_required
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.message.models import Message, MessageAttachment
-from ietf.person.models import Person
+from ietf.person.models import Person, Email
 from ietf.submit.forms import ( SubmissionManualUploadForm, SubmissionAutoUploadForm, AuthorForm,
     SubmitterForm, EditSubmissionForm, PreapprovalForm, ReplacesForm, SubmissionEmailForm, MessageModelForm )
 from ietf.submit.mail import ( send_full_url, send_manual_post_request, add_submission_email, get_reply_to )
@@ -103,10 +103,25 @@ def api_submit(request):
                 username = form.cleaned_data['user']
                 user = User.objects.filter(username=username)
                 if user.count() == 0:
-                    return err(400, "No such user: %s" % username)
+                    # See if a secondary login was being used
-                if user.count() > 1:
+                    email = Email.objects.filter(address=username, active=True)
+                    # The error messages don't talk about 'email', as the field we're
+                    # looking at is still the 'username' field.
+                    if email.count() == 0:
+                        return err(400, "No such user: %s" % username)
+                    elif email.count() > 1:
+                        return err(500, "Multiple matching accounts for %s" % username)
+                    email = email.first()
+                    if not hasattr(email, 'person'):
+                        return err(400, "No person matches %s" % username)
+                    person = email.person
+                    if not hasattr(person, 'user'):
+                        return err(400, "No user matches: %s" % username)
+                    user = person.user
+                elif user.count() > 1:
                     return err(500, "Multiple matching accounts for %s" % username)
-                user = user.first()
+                else:
+                    user = user.first()
                 if not hasattr(user, 'person'):
                     return err(400, "No person with username %s" % username)
 
@@ -130,7 +145,7 @@ def api_submit(request):
                 if errors:
                     raise ValidationError(errors)
 
-                if not user.username.lower() in [ a['email'].lower() for a in authors ]:
+                if not username.lower() in [ a['email'].lower() for a in authors ]:
                     raise ValidationError('Submitter %s is not one of the document authors' % user.username)
 
                 submission.submitter = user.person.formatted_email()

ietf/sync/iana.py

@@ -7,8 +7,7 @@ import datetime
 import email
 import json
 import re
-
+import requests
-from urllib.request import Request, urlopen
 
 from django.conf import settings
 from django.utils.encoding import smart_bytes, force_str
@@ -21,19 +20,12 @@ from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person
 from ietf.utils.mail import parseaddr
-from ietf.utils.text import decode
 from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone
 
 
 #PROTOCOLS_URL = "https://www.iana.org/protocols/"
 #CHANGES_URL = "https://datatracker.dev.icann.org:8080/data-tracker/changes"
 
-def fetch_protocol_page(url):
-    f = urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
-    text = decode(f.read())
-    f.close()
-    return text
-    
 def parse_protocol_page(text):
     """Parse IANA protocols page to extract referenced RFCs (as
     rfcXXXX document names)."""
@@ -73,14 +65,11 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than):
 def fetch_changes_json(url, start, end):
     url += "?start=%s&end=%s" % (urlquote(local_timezone_to_utc(start).strftime("%Y-%m-%d %H:%M:%S")),
                                  urlquote(local_timezone_to_utc(end).strftime("%Y-%m-%d %H:%M:%S")))
-    request = Request(url)
     # HTTP basic auth
     username = "ietfsync"
     password = settings.IANA_SYNC_PASSWORD
-    request.add_header("Authorization", "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""))
+    headers = { "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") }
-    f = urlopen(request)
+    text = requests.get(url, headers=headers).text
-    text = decode(f.read())
-    f.close()
     return text
 
 def parse_changes_json(text):

ietf/sync/rfceditor.py

@@ -5,13 +5,13 @@
 import base64
 import datetime
 import re
+import requests
 
-from urllib.request import Request, urlopen
 from urllib.parse import urlencode
 from xml.dom import pulldom, Node
 
 from django.conf import settings
-from django.utils.encoding import smart_bytes, force_str
+from django.utils.encoding import smart_bytes, force_str, force_text
 
 import debug                            # pyflakes:ignore
 
@@ -24,7 +24,6 @@ from ietf.name.models import StdLevelName, StreamName
 from ietf.person.models import Person
 from ietf.utils.log import log
 from ietf.utils.mail import send_mail_text
-from ietf.utils.text import decode
 
 #QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 #INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
@@ -527,28 +526,28 @@ def post_approved_draft(url, name):
     the data from the Datatracker and start processing it. Returns
     response and error (empty string if no error)."""
 
-    request = Request(url)
-    request.add_header("Content-type", "application/x-www-form-urlencoded")
-    request.add_header("Accept", "text/plain")
     # HTTP basic auth
     username = "dtracksync"
     password = settings.RFC_EDITOR_SYNC_PASSWORD
-    request.add_header("Authorization", "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""))
+    headers = {
+            "Content-type": "application/x-www-form-urlencoded",
+            "Accept": "text/plain",
+            "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""),
+        }
 
     log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url))
     text = error = ""
+
     try:
-        f = urlopen(request, data=smart_bytes(urlencode({ 'draft': name })), timeout=20)
+        r = requests.post(url, headers=headers, data=smart_bytes(urlencode({ 'draft': name })), timeout=20)
-        text = decode(f.read())
-        status_code = f.getcode()
-        f.close()
-        log("RFC-Editor notification result for draft '%s': %s:'%s'" % (name, status_code, text))
 
-        if status_code != 200:
+        log("RFC-Editor notification result for draft '%s': %s:'%s'" % (name, r.status_code, r.text))
-            raise RuntimeError("Status code is not 200 OK (it's %s)." % status_code)
 
-        if text != "OK":
+        if r.status_code != 200:
-            raise RuntimeError("Response is not \"OK\".")
+            raise RuntimeError("Status code is not 200 OK (it's %s)." % r.status_code)
+
+        if force_text(r.text) != "OK":
+            raise RuntimeError('Response is not "OK" (it\'s "%s").' % r.text)
 
     except Exception as e:
         # catch everything so we don't leak exceptions, convert them

ietf/sync/views.py

@@ -77,7 +77,7 @@ def notify(request, org, notification):
 
     if request.method == "POST":
         def runscript(name):
-            python = os.path.join(settings.BASE_DIR, "env", "bin", "python")
+            python = os.path.join(os.path.dirname(settings.BASE_DIR), "env", "bin", "python")
             cmd = [python, os.path.join(SYNC_BIN_PATH, name)]
             cmdstring = " ".join(cmd)
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

ietf/templates/doc/document_draft.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{# Copyright The IETF Trust 2016, All Rights Reserved #}
+{# Copyright The IETF Trust 2016-2020, All Rights Reserved #}
 {% load origin %}
 {% load staticfiles %}
 {% load ietf_filters %}
@@ -52,10 +52,10 @@
             RFC - {{ doc.std_level }}
             ({% if published %}{{ published.time|date:"F Y" }}{% else %}publication date unknown{% endif %}{% if has_errata %}; <a href="https://www.rfc-editor.org/errata_search.php?rfc={{ rfc_number }}" rel="nofollow">Errata</a>{% else %}; No errata{% endif %})
 
-            {% if obsoleted_by %}<div>Obsoleted by {{ obsoleted_by|join:", "|urlize_ietf_docs }}</div>{% endif %}
+            {% if obsoleted_by %}<div>Obsoleted by {{ obsoleted_by|urlize_doc_list|join:", " }}</div>{% endif %}
-            {% if updated_by %}<div>Updated by {{ updated_by|join:", "|urlize_ietf_docs }}</div>{% endif %}
+            {% if updated_by %}<div>Updated by {{ updated_by|urlize_doc_list|join:", " }}</div>{% endif %}
-            {% if obsoletes %}<div>Obsoletes {{ obsoletes|join:", "|urlize_ietf_docs }}</div>{% endif %}
+            {% if obsoletes %}<div>Obsoletes {{ obsoletes|urlize_doc_list|join:", " }}</div>{% endif %}
-            {% if updates %}<div>Updates {{ updates|join:", "|urlize_ietf_docs }}</div>{% endif %}
+            {% if updates %}<div> Updates {{ updates|urlize_doc_list|join:", " }}</div>{% endif %}
             {% if status_changes %}<div>Status changed by {{ status_changes|join:", "|urlize_ietf_docs }}</div>{% endif %}
             {% if proposed_status_changes %}<div>Proposed status changed by {{ proposed_status_changes|join:", "|urlize_ietf_docs }}</div>{% endif %}
             {% if rfc_aliases %}<div>Also known as {{ rfc_aliases|join:", "|urlize_ietf_docs }}</div>{% endif %}
@@ -89,7 +89,7 @@
             {% endif %}
           </td>
           <td>
-            {{ replaces|join:", "|urlize_ietf_docs|default:"(None)" }}
+            {{ replaces|urlize_doc_list|join:", "|default:"(None)" }}
           </td>
         </tr>
       {% endif %}
@@ -100,7 +100,7 @@
           <th>Replaced by</th>
           <td class="edit"></td>
           <td>
-            {{ replaced_by|join:", "|urlize_ietf_docs }}
+            {{ replaced_by|urlize_doc_list|join:", " }}
           </td>
         </tr>
       {% endif %}
@@ -116,7 +116,7 @@
                 {% endif %}
               </td>
               <td>
-                {{ possibly_replaces|join:", "|urlize_ietf_docs }}
+                {{ possibly_replaces|urlize_doc_list|join:", " }}
               </td>
             </tr>
           {% endif %}
@@ -131,7 +131,7 @@
                 {% endif %}
               </td>
               <td>
-                {{ possibly_replaced_by|join:", "|urlize_ietf_docs }}
+                {{ possibly_replaced_by|urlize_doc_list|join:", " }}
               </td>
             </tr>
           {% endif %}

ietf/templates/doc/search/search_result_row.html

@@ -2,6 +2,7 @@
 {% load widget_tweaks %}
 {% load ietf_filters %}
 {% load ballot_icon %}
+{% load person_filters %}
 
 <tr {% spaceless %}
     {% if color_row_positions %}
@@ -120,9 +121,9 @@
   {% if ad_name == None or ad_name != doc.ad.plain_name %}
     <td class="area-director">
       {% if doc.ad %}
-        <a title="Area Director" href="mailto:{{ doc.ad.email_address|urlencode }}">{{ doc.ad }}</a><br>
+        {% person_link doc.ad title="Area Director" %}<br>
       {% endif %}
-      {% if doc.shepherd %}<a title="Shepherd" href="mailto:{{doc.shepherd}}"><small class="text-muted">{{doc.shepherd.person.name}}</small></a>{% endif %}
+      {% if doc.shepherd %}{% email_person_link doc.shepherd title="Shepherd" class="small text-muted" %}{% endif %}
     </td>
   {% endif %}