Replaced most cases of using of urlopen(), instead using the higher-level 'requests' module where it simplifies the code.

- Legacy-Id: 17380

PLAN

@@ -7,13 +7,6 @@ Updated: $Date$
 Planned work in rough order
 ===========================
 
-* Refactor the use of urlopen().  This is triggered by the transition to
-  python 3: All instances of .read() from urlopen() objects return byte
-  streams under Python 3, while the code wants to deal with str objects, which
-  are unicode under Python 3.  Instead of building our own adapter to extract
-  the charset and do the decoding, we should transition to the 'requests'
-  module, which provides decoded unicode text directly from the response.
-
 * Transition to Django 2.x (depends on Python 3.x).  Security updates to
   Django 1.11 will cease around April 2020.
 

ietf/doc/tests_ballot.py

@@ -619,10 +619,10 @@ class BallotWriteupsTests(TestCase):
             verify_can_see(username, url)
 
 class ApproveBallotTests(TestCase):
-    @mock.patch('ietf.sync.rfceditor.urlopen', autospec=True)
+    @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True)
     def test_approve_ballot(self, mock_urlopen):
-        mock_urlopen.return_value.read = lambda : b'OK'
-        mock_urlopen.return_value.getcode = lambda :200
+        mock_urlopen.return_value.text = b'OK'
+        mock_urlopen.return_value.status_code = 200
         #
         ad = Person.objects.get(name="Areað Irector")
         draft = IndividualDraftFactory(ad=ad, intended_std_level_id='ps')

ietf/doc/tests_draft.py

@@ -1199,10 +1199,10 @@ class SubmitToIesgTests(TestCase):
 
 
 class RequestPublicationTests(TestCase):
-    @mock.patch('ietf.sync.rfceditor.urlopen', autospec=True)
-    def test_request_publication(self, mock_urlopen):
-        mock_urlopen.return_value.read = lambda : b'OK'
-        mock_urlopen.return_value.getcode = lambda :200
+    @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True)
+    def test_request_publication(self, mockobj):
+        mockobj.return_value.text = b'OK'
+        mockobj.return_value.status_code = 200
         #
         draft = IndividualDraftFactory(stream_id='iab',group__acronym='iab',intended_std_level_id='inf',states=[('draft-stream-iab','approved')])
 

ietf/meeting/tests_views.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import datetime
 import io
+import json
 import os
 import random
 import re
@@ -436,9 +437,10 @@ class MeetingTests(TestCase):
         response = self.client.get(url)
         self.assertContains(response, 'test acknowledgements')
 
-    @patch('six.moves.urllib.request.urlopen')
-    def test_proceedings_attendees(self, mock_urlopen):
-        mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
+    @patch('ietf.meeting.utils.requests.get')
+    def test_proceedings_attendees(self, mockobj):
+        mockobj.return_value.text = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
+        mockobj.return_value.json = lambda: json.loads(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
         make_meeting_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96")
         finalize(meeting)

ietf/meeting/utils.py

@@ -1,12 +1,11 @@
-# Copyright The IETF Trust 2016-2019, All Rights Reserved
+# Copyright The IETF Trust 2016-2020, All Rights Reserved
 # -*- coding: utf-8 -*-
 
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import datetime
-import json
-import six.moves.urllib.request
+import requests
 
 from six.moves.urllib.error import HTTPError
 from django.conf import settings
@@ -115,7 +114,7 @@ def create_proceedings_templates(meeting):
     # Get meeting attendees from registration system
     url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number)
     try:
-        attendees = json.load(six.moves.urllib.request.urlopen(url))
+        attendees = requests.get(url).json()
     except (ValueError, HTTPError):
         attendees = []
 

ietf/sync/iana.py

@@ -9,8 +9,7 @@ import datetime
 import email
 import json
 import re
-
-from six.moves.urllib.request import Request, urlopen
+import requests
 
 from django.conf import settings
 from django.utils.encoding import smart_bytes, force_str
@@ -23,19 +22,12 @@ from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person
 from ietf.utils.mail import parseaddr
-from ietf.utils.text import decode
 from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone
 
 
 #PROTOCOLS_URL = "https://www.iana.org/protocols/"
 #CHANGES_URL = "https://datatracker.dev.icann.org:8080/data-tracker/changes"
 
-def fetch_protocol_page(url):
-    f = urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
-    text = decode(f.read())
-    f.close()
-    return text
-    
 def parse_protocol_page(text):
     """Parse IANA protocols page to extract referenced RFCs (as
     rfcXXXX document names)."""
@@ -75,14 +67,11 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than):
 def fetch_changes_json(url, start, end):
     url += "?start=%s&end=%s" % (urlquote(local_timezone_to_utc(start).strftime("%Y-%m-%d %H:%M:%S")),
                                  urlquote(local_timezone_to_utc(end).strftime("%Y-%m-%d %H:%M:%S")))
-    request = Request(url)
     # HTTP basic auth
     username = "ietfsync"
     password = settings.IANA_SYNC_PASSWORD
-    request.add_header("Authorization", "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""))
-    f = urlopen(request)
-    text = decode(f.read())
-    f.close()
+    headers = { "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") }
+    text = requests.get(url, headers=headers).text
     return text
 
 def parse_changes_json(text):

ietf/sync/rfceditor.py

@@ -7,14 +7,14 @@ from __future__ import absolute_import, print_function, unicode_literals
 import base64
 import datetime
 import re
+import requests
 import six
 
-from six.moves.urllib.request import Request, urlopen
 from six.moves.urllib.parse import urlencode
 from xml.dom import pulldom, Node
 
 from django.conf import settings
-from django.utils.encoding import smart_bytes, force_str
+from django.utils.encoding import smart_bytes, force_str, force_text
 
 import debug                            # pyflakes:ignore
 
@@ -27,7 +27,6 @@ from ietf.name.models import StdLevelName, StreamName
 from ietf.person.models import Person
 from ietf.utils.log import log
 from ietf.utils.mail import send_mail_text
-from ietf.utils.text import decode
 
 #QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 #INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
@@ -530,28 +529,28 @@ def post_approved_draft(url, name):
     the data from the Datatracker and start processing it. Returns
     response and error (empty string if no error)."""
 
-    request = Request(url)
-    request.add_header("Content-type", "application/x-www-form-urlencoded")
-    request.add_header("Accept", "text/plain")
     # HTTP basic auth
     username = "dtracksync"
     password = settings.RFC_EDITOR_SYNC_PASSWORD
-    request.add_header("Authorization", "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""))
+    headers = {
+            "Content-type": "application/x-www-form-urlencoded",
+            "Accept": "text/plain",
+            "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""),
+        }
 
     log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url))
     text = error = ""
+
     try:
-        f = urlopen(request, data=smart_bytes(urlencode({ 'draft': name })), timeout=20)
-        text = decode(f.read())
-        status_code = f.getcode()
-        f.close()
-        log("RFC-Editor notification result for draft '%s': %s:'%s'" % (name, status_code, text))
+        r = requests.post(url, headers=headers, data=smart_bytes(urlencode({ 'draft': name })), timeout=20)
 
-        if status_code != 200:
-            raise RuntimeError("Status code is not 200 OK (it's %s)." % status_code)
+        log("RFC-Editor notification result for draft '%s': %s:'%s'" % (name, r.status_code, r.text))
 
-        if text != "OK":
-            raise RuntimeError("Response is not \"OK\".")
+        if r.status_code != 200:
+            raise RuntimeError("Status code is not 200 OK (it's %s)." % r.status_code)
+
+        if force_text(r.text) != "OK":
+            raise RuntimeError('Response is not "OK" (it\'s "%s").' % r.text)
 
     except Exception as e:
         # catch everything so we don't leak exceptions, convert them