From cf4a4b02a73861a4d25c9eee178a0ebffa5a9e8c Mon Sep 17 00:00:00 2001
From: Henrik Levkowetz <henrik@levkowetz.com>
Date: Sat, 18 Feb 2017 21:50:18 +0000
Subject: [PATCH] Reworked the email address handling in order to be able to
 support non-ascii names as part of email address fields.  Reworked the
 generation of user names in the test suite to generate names from multiple
 non-ascii locales.   Fixes issue #2080.  - Legacy-Id: 12872

---
 ietf/group/models.py                    | 10 +++++--
 ietf/idindex/index.py                   |  4 +--
 ietf/mailtrigger/models.py              |  8 ++++--
 ietf/nomcom/templatetags/nomcom_tags.py | 16 +++++------
 ietf/nomcom/tests.py                    |  2 +-
 ietf/person/factories.py                | 12 ++++++---
 ietf/person/fields.py                   |  3 ++-
 ietf/person/models.py                   | 20 +++++++++++++-
 ietf/person/tests.py                    |  3 ++-
 ietf/person/utils.py                    |  3 ++-
 ietf/submit/tests.py                    |  4 +--
 ietf/utils/mail.py                      | 36 ++++++++++++++++++-------
 ietf/utils/text.py                      |  7 +++++
 13 files changed, 95 insertions(+), 33 deletions(-)

diff --git a/ietf/group/models.py b/ietf/group/models.py
index bb6179677..c67e1a270 100644
--- a/ietf/group/models.py
+++ b/ietf/group/models.py
@@ -1,15 +1,18 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 
 import datetime
+import email.utils
 from urlparse import urljoin
 
 from django.db import models
 
+import debug                            # pyflakes:ignore
+
 from ietf.group.colors import fg_group_colors, bg_group_colors
 from ietf.name.models import GroupStateName, GroupTypeName, DocTagName, GroupMilestoneStateName, RoleName
 from ietf.person.models import Email, Person
+from ietf.utils.mail import formataddr
 
-import debug                            # pyflakes:ignore
 
 class GroupInfo(models.Model):
     time = models.DateTimeField(default=datetime.datetime.now)
@@ -254,8 +257,11 @@ class Role(models.Model):
     def __unicode__(self):
         return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name)
 
+    def formatted_ascii_email(self):
+        return email.utils.formataddr((self.person.plain_ascii(), self.email.address))
+
     def formatted_email(self):
-        return u'"%s" <%s>' % (self.person.plain_name(), self.email.address)
+        return formataddr((self.person.plain_name(), self.email.address))
 
 class RoleHistory(models.Model):
     # RoleHistory doesn't have a time field as it's not supposed to be
diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py
index 5ccecba02..2601f9888 100644
--- a/ietf/idindex/index.py
+++ b/ietf/idindex/index.py
@@ -127,9 +127,9 @@ def all_id2_txt():
         else:
             l.append(a.author.person.plain_name())
 
-    shepherds = dict((e.pk, e.formatted_email().replace('"', ''))
+    shepherds = dict((e.pk, e.formatted_ascii_email().replace('"', ''))
                      for e in Email.objects.filter(shepherd_document_set__type="draft").select_related("person").distinct())
-    ads = dict((p.pk, p.formatted_email().replace('"', ''))
+    ads = dict((p.pk, p.formatted_ascii_email().replace('"', ''))
                for p in Person.objects.filter(ad_document_set__type="draft").distinct())
 
     res = []
diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py
index 526b5f5dd..1ab77baf3 100644
--- a/ietf/mailtrigger/models.py
+++ b/ietf/mailtrigger/models.py
@@ -4,6 +4,10 @@ from django.db import models
 from django.template import Template, Context
 
 from email.utils import parseaddr
+from ietf.utils.mail import formataddr
+
+
+import debug                            # pyflakes:ignore
 
 from ietf.group.models import Role
 
@@ -14,7 +18,7 @@ def clean_duplicates(addrlist):
         if (name,addr)==('',''):
             retval.add(a)
         elif name:
-            retval.add('"%s" <%s>'%(name,addr))
+            retval.add(formataddr((name,addr)))
         else:
             retval.add(addr)
     return list(retval) 
@@ -200,7 +204,7 @@ class Recipient(models.Model):
             doc=submission.existing_document()
             if doc:
                 old_authors = [i.author.formatted_email() for i in doc.documentauthor_set.all() if not i.author.invalid_address()]
-                new_authors = [u'"%s" <%s>' % (author["name"], author["email"]) for author in submission.authors_parsed() if author["email"]]
+                new_authors = [ formataddr((author["name"], author["email"])) for author in submission.authors_parsed() if author["email"]]
                 addrs.extend(old_authors)
                 if doc.group and set(old_authors)!=set(new_authors):
                     if doc.group.type_id in ['wg','rg','ag']:
diff --git a/ietf/nomcom/templatetags/nomcom_tags.py b/ietf/nomcom/templatetags/nomcom_tags.py
index 0af233808..6abd90671 100644
--- a/ietf/nomcom/templatetags/nomcom_tags.py
+++ b/ietf/nomcom/templatetags/nomcom_tags.py
@@ -5,15 +5,15 @@ from django import template
 from django.conf import settings
 from django.template.defaultfilters import linebreaksbr, force_escape
 
-from ietf.utils.pipe import pipe
-from ietf.utils.log import log
-from ietf.doc.templatetags.ietf_filters import wrap_text
-
-from ietf.person.models import Person
-from ietf.nomcom.utils import get_nomcom_by_year, retrieve_nomcom_private_key
-
 import debug           # pyflakes:ignore
 
+from ietf.doc.templatetags.ietf_filters import wrap_text
+from ietf.nomcom.utils import get_nomcom_by_year, retrieve_nomcom_private_key
+from ietf.person.models import Person
+from ietf.utils.log import log
+from ietf.utils.mail import formataddr
+from ietf.utils.pipe import pipe
+
 
 register = template.Library()
 
@@ -41,7 +41,7 @@ def formatted_email(address):
         persons = Person.objects.filter(email__address__in=[address])
         person = persons and persons[0] or None
     if person and person.name:
-        return u'"%s" <%s>' % (person.plain_name(), address)
+        return formataddr((person.plain_name(), address))
     else:
         return address
 
diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py
index 0dc9a00c3..8c754eef6 100644
--- a/ietf/nomcom/tests.py
+++ b/ietf/nomcom/tests.py
@@ -1705,7 +1705,7 @@ Junk body for testing
                                          'duplicate_persons':[nominee2.person.pk]})
         self.assertEqual(response.status_code, 302)
         self.assertEqual(len(outbox),1)
-        self.assertTrue(all([str(x.person.pk) in unicode(outbox[0]) for x in [nominee1,nominee2]]))
+        self.assertTrue(all([str(x.person.pk) in outbox[0].get_payload(decode=True) for x in [nominee1,nominee2]]))
 
 
 class NomComIndexTests(TestCase):
diff --git a/ietf/person/factories.py b/ietf/person/factories.py
index 57e88cd82..34ce2a7e0 100644
--- a/ietf/person/factories.py
+++ b/ietf/person/factories.py
@@ -2,11 +2,15 @@ import os
 import factory
 import faker 
 import shutil
+import random
+import faker.config
 from unidecode import unidecode
 
 from django.conf import settings
 from django.contrib.auth.models import User
 
+import debug                            # pyflakes:ignore
+
 from ietf.person.models import Person, Alias, Email
 
 fake = faker.Factory.create()
@@ -15,10 +19,12 @@ class UserFactory(factory.DjangoModelFactory):
     class Meta:
         model = User
         django_get_or_create = ('username',)
+        exclude = ['locale', ]
 
-    first_name = factory.Faker('first_name')
-    last_name = factory.Faker('last_name')
-    email = factory.LazyAttributeSequence(lambda u, n: '%s.%s_%d@%s'%(u.first_name,u.last_name,n,fake.domain_name()))
+    locale = random.sample(faker.config.AVAILABLE_LOCALES, 1)[0]
+    first_name = factory.Faker('first_name', locale)
+    last_name = factory.Faker('last_name', locale)
+    email = factory.LazyAttributeSequence(lambda u, n: '%s.%s_%d@%s'%(unidecode(u.first_name),unidecode(u.last_name),n, fake.domain_name()))
     username = factory.LazyAttribute(lambda u: u.email)
 
     @factory.post_generation
diff --git a/ietf/person/fields.py b/ietf/person/fields.py
index d7ba971b3..fbf21d30d 100644
--- a/ietf/person/fields.py
+++ b/ietf/person/fields.py
@@ -1,4 +1,5 @@
 import json
+import six
 
 from collections import Counter
 from urllib import urlencode
@@ -108,7 +109,7 @@ class SearchablePersonsField(forms.CharField):
             #if self.only_users:
             #    objs = objs.exclude(person__user=None)
 
-        found_pks = [str(o.pk) for o in objs]
+        found_pks = [ six.text_type(o.pk) for o in objs]
         failed_pks = [x for x in pks if x not in found_pks]
         if failed_pks:
             raise forms.ValidationError(u"Could not recognize the following {model_name}s: {pks}. You can only input {model_name}s already registered in the Datatracker.".format(pks=", ".join(failed_pks), model_name=self.model.__name__.lower()))
diff --git a/ietf/person/models.py b/ietf/person/models.py
index 9713ccb25..f07dd3323 100644
--- a/ietf/person/models.py
+++ b/ietf/person/models.py
@@ -1,6 +1,8 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 
 import datetime
+import email.utils
+import email.header
 from hashids import Hashids
 from unidecode import unidecode
 from urlparse import urljoin
@@ -17,6 +19,8 @@ import debug                            # pyflakes:ignore
 from ietf.person.name import name_parts, initials
 from ietf.utils.mail import send_mail_preformatted
 from ietf.utils.storage import NoLocationMigrationFileSystemStorage
+from ietf.utils.mail import formataddr
+
 
 class PersonInfo(models.Model):
     time = models.DateTimeField(default=datetime.datetime.now)      # When this Person record entered the system
@@ -106,6 +110,14 @@ class PersonInfo(models.Model):
             return e.address
         else:
             return ""
+    def formatted_ascii_email(self):
+        e = self.email_set.filter(primary=True).first()
+        if not e:
+            e = self.email_set.order_by("-active", "-time").first()
+        if e:
+            return e.formatted_ascii_email()
+        else:
+            return ""
     def formatted_email(self):
         e = self.email_set.filter(primary=True).first()
         if not e:
@@ -225,9 +237,15 @@ class Email(models.Model):
     def get_name(self):
         return self.person.plain_name() if self.person else self.address
 
+    def formatted_ascii_email(self):
+        if self.person:
+                return email.utils.formataddr((self.person.plain_ascii(), self.address))
+        else:
+            return self.address
+
     def formatted_email(self):
         if self.person:
-            return u'"%s" <%s>' % (self.person.plain_ascii(), self.address)
+            return formataddr((self.person.plain_name(), self.address))
         else:
             return self.address
 
diff --git a/ietf/person/tests.py b/ietf/person/tests.py
index 027884642..4b8d21f34 100644
--- a/ietf/person/tests.py
+++ b/ietf/person/tests.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 
 import json
 from pyquery import PyQuery
@@ -41,7 +42,7 @@ class PersonTests(TestCase):
         url = urlreverse("ietf.person.views.profile", kwargs={ "email_or_name": person.plain_name()})
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertIn(person.photo_name(), r.content)
+        self.assertIn(person.photo_name(), r.content.decode(r.charset))
         q = PyQuery(r.content)
         self.assertIn("Photo of %s"%person, q("div.bio-text img.bio-photo").attr("alt"))
 
diff --git a/ietf/person/utils.py b/ietf/person/utils.py
index 55e7a6929..f541ef652 100755
--- a/ietf/person/utils.py
+++ b/ietf/person/utils.py
@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 import pprint 
 
 from django.contrib import admin
@@ -18,7 +19,7 @@ def merge_persons(source,target,stream):
         if alias.name in target_aliases:
             alias.delete()
         else:
-            print >>stream,"Merging alias: {}".format(alias.name)
+            print >>stream, "Merging alias: {}".format(alias.name)
             alias.person = target
             alias.save()
     
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index a767e7c34..1926632bc 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -358,8 +358,8 @@ class SubmitTests(TestCase):
                 if stream_type=='ise':
                    self.assertTrue("rfc-ise@" in confirm_email["To"].lower())
         else:
-            self.assertTrue("chairs have been copied" not in unicode(confirm_email))
-            self.assertTrue("mars-chairs@" not in confirm_email["To"].lower())
+            self.assertNotIn("chairs have been copied", unicode(confirm_email))
+            self.assertNotIn("mars-chairs@", confirm_email["To"].lower())
 
         confirm_url = self.extract_confirm_url(confirm_email)
 
diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py
index f508c8b18..3878a7031 100644
--- a/ietf/utils/mail.py
+++ b/ietf/utils/mail.py
@@ -1,6 +1,14 @@
 # Copyright The IETF Trust 2007, All Rights Reserved
 
-from email.utils import make_msgid, formatdate, formataddr, parseaddr, getaddresses
+import copy
+import datetime
+import smtplib
+import sys
+import textwrap
+import time
+import traceback
+
+from email.utils import make_msgid, formatdate, formataddr as simple_formataddr, parseaddr, getaddresses
 from email.mime.text import MIMEText
 from email.mime.message import MIMEMessage
 from email.mime.multipart import MIMEMultipart
@@ -8,20 +16,17 @@ from email.header import Header
 from email import message_from_string
 from email import charset as Charset
 
-import smtplib
 from django.conf import settings
 from django.contrib import messages
 from django.core.exceptions import ImproperlyConfigured
 from django.template.loader import render_to_string
 from django.template import Context,RequestContext
+
+import debug                            # pyflakes:ignore
+
 import ietf
 from ietf.utils.log import log
-import sys
-import time
-import copy
-import textwrap
-import traceback
-import datetime
+from ietf.utils.text import isascii
 
 # Testing mode:
 # import ietf.utils.mail
@@ -189,9 +194,22 @@ def send_mail_text(request, to, frm, subject, txt, cc=None, extra=None, toUser=F
     msg = encode_message(txt)
     return send_mail_mime(request, to, frm, subject, msg, cc, extra, toUser, bcc)
         
+def formataddr(addrtuple):
+    """
+    Takes a name and email address, and inspects the name to see if it needs
+    to be encoded in an email.header.Header before being used in an email.message
+    address field.  Does what's needed, and returns a string value suitable for
+    use in a To: or Cc: email header field.
+    """
+    name, addr = addrtuple
+    if name and not isascii(name):
+        name = str(Header(name, 'utf-8'))
+    return simple_formataddr((name, addr))
+
 def condition_message(to, frm, subject, msg, cc, extra):
+
     if isinstance(frm, tuple):
-	frm = formataddr(frm)
+        frm = formataddr(frm)
     if isinstance(to, list) or isinstance(to, tuple):
         to = ", ".join([isinstance(addr, tuple) and formataddr(addr) or addr for addr in to if addr])
     if isinstance(cc, list) or isinstance(cc, tuple):
diff --git a/ietf/utils/text.py b/ietf/utils/text.py
index a4a13d6e6..f9211d21e 100644
--- a/ietf/utils/text.py
+++ b/ietf/utils/text.py
@@ -49,3 +49,10 @@ def fill(text, width):
             wrapped.append(para)
     return "\n\n".join(wrapped)
         
+def isascii(text):
+    try:
+        text.encode('ascii')
+        return True
+    except UnicodeEncodeError:
+        return False
+