altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Merged in the latest GDPR changes. This refines the handling of the consent checkbox on the account page; refines the Consent Needed warning given on login if consent is needed; tweaks several models to set the on_deletion fields for FK to User and Person appropriately; adds a Person.needs_consent() method to capture the logic of which fields require consent; refines the Person.plain_name() method and the user.log.log() function; and adds 2 management commands to send out consent requests and delete non-consent information, respectively.

Browse source 
- Legacy-Id: 15464
This commit is contained in:
Henrik Levkowetz 2018-09-16 23:12:43 +00:00
parent a389e24fb6 b85e1c46d9
commit 4fba531e9e
11 changed files with 357 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ietf/ietfauth/forms.py
View file

@ -116,8 +116,6 @@ def get_person_form(*args, **kwargs):
if f in self.fields:
self.fields[f].label += ' \u2020'
self.fields["consent"].required = True
self.unidecoded_ascii = False
if self.data and not self.data.get("ascii", "").strip():
@ -150,8 +148,13 @@ def get_person_form(*args, **kwargs):
def clean_consent(self):
consent = self.cleaned_data.get('consent')
if consent == False:
raise forms.ValidationError("In order to modify your profile data, you must permit the IETF to use the uploaded data.")
require_consent = (
self.cleaned_data.get('name') != person.name_from_draft
or self.cleaned_data.get('ascii') != person.name_from_draft
or self.cleaned_data.get('biography')
)
if consent == False and require_consent:
raise forms.ValidationError("In order to modify your profile with data that require consent, you must permit the IETF to use the uploaded data.")
return consent
return PersonForm(*args, **kwargs)

10
ietf/ietfauth/views.py
View file

@ -580,12 +580,14 @@ def login(request, extra_context=None):
which is not recognized as a valid password hash.
"""
require_consent = []
if request.method == "POST":
form = AuthenticationForm(request, data=request.POST)
username = form.data.get('username')
user = User.objects.filter(username=username).first()
#
require_consent = []
if user.person and not user.person.consent:
require_consent = user.person.needs_consent()
if user:
if hasattr(user, 'person') and not user.person.consent:
person = user.person
@ -618,8 +620,10 @@ def login(request, extra_context=None):
You have personal information associated with your account which is not
derived from draft submissions or other ietf work, namely: %s. Please go
to your <a href='/accounts/profile'>account profile</a> and review your
personal information, and confirm that it may be used and displayed
within the IETF datatracker.
personal information, then scoll to the bottom and check the 'confirm'
checkbox and submit the form, in order to to indicate that that the
provided personal information may be used and displayed within the IETF
datatracker.
""" % ', '.join(require_consent)))
return response

4
ietf/nomcom/models.py
View file

@ -90,7 +90,7 @@ class Nomination(models.Model):
nominee = ForeignKey('Nominee')
comments = ForeignKey('Feedback')
nominator_email = models.EmailField(verbose_name='Nominator Email', blank=True)
user = ForeignKey(User, editable=False)
user = ForeignKey(User, editable=False, null=True, on_delete=models.SET_NULL)
time = models.DateTimeField(auto_now_add=True)
share_nominator = models.BooleanField(verbose_name='Share nominator name with candidate', default=False,
help_text='Check this box to allow the NomCom to let the '
@ -247,7 +247,7 @@ class Feedback(models.Model):
subject = models.TextField(verbose_name='Subject', blank=True)
comments = EncryptedTextField(verbose_name='Comments')
type = ForeignKey(FeedbackTypeName, blank=True, null=True)
user = ForeignKey(User, editable=False, blank=True, null=True)
user = ForeignKey(User, editable=False, blank=True, null=True, on_delete=models.SET_NULL)
time = models.DateTimeField(auto_now_add=True)
objects = FeedbackManager()

23
ietf/person/migrations/0006_auto_20180910_0719.py Normal file
View file

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.13 on 2018-09-10 07:19
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations
import django.db.models.deletion
import ietf.utils.models
class Migration(migrations.Migration):
dependencies = [
('person', '0005_populate_person_name_from_draft'),
]
operations = [
migrations.AlterField(
model_name='person',
name='user',
field=ietf.utils.models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
),
]

28
ietf/person/models.py
View file

@ -12,6 +12,7 @@ from urlparse import urljoin
from django.conf import settings
from django.core.validators import validate_email
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.contrib.auth.models import User
from django.template.loader import render_to_string
@ -31,7 +32,7 @@ from ietf.utils.models import ForeignKey, OneToOneField
class Person(models.Model):
history = HistoricalRecords()
user = OneToOneField(User, blank=True, null=True)
user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL)
time = models.DateTimeField(default=datetime.datetime.now) # When this Person record entered the system
# The normal unicode form of the name. This must be
# set to the same value as the ascii-form if equal.
@ -163,6 +164,31 @@ class Person(models.Model):
from ietf.doc.models import Document
return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time')
def needs_consent(self):
"""
Returns an empty list or a list of fields which holds information that
requires consent to be given.
"""
needs_consent = []
if self.name != self.name_from_draft:
needs_consent.append("full name")
if self.ascii != self.name_from_draft:
needs_consent.append("ascii name")
if self.biography and not (self.role_set.exists() or self.rolehistory_set.exists()):
needs_consent.append("biography")
if self.user_id:
needs_consent.append("login")
try:
if self.user.communitylist_set.exists():
needs_consent.append("draft notification subscription(s)")
except ObjectDoesNotExist:
pass
for email in self.email_set.all():
if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]:
needs_consent.append("email address(es)")
break
return needs_consent
def save(self, *args, **kwargs):
created = not self.pk
super(Person, self).save(*args, **kwargs)

2
ietf/person/name.py
View file

@ -71,7 +71,7 @@ def initials(name):
def plain_name(name):
prefix, first, middle, last, suffix = name_parts(name)
return u" ".join([first, last])
return u" ".join( n for n in (first, last) if n)
def capfirst(s):
# Capitalize the first word character, skipping non-word characters and

23
ietf/templates/utils/personal_information_notice.txt Normal file
View file

@ -0,0 +1,23 @@
{% load ietf_filters %}{% filter wordwrap:78 %}
Dear {{ person.plain_name }},
This email is regarding some of the personal information stored in your IETF datatracker
profile; information for which we require your consent for storage and use.
If you do nothing in response to this email, the information in your profile
that requires consent ({{ fields|safe }} and login) will be deleted {{ days }}
days from now, on {{ date }}. If you later wish to create a new login, you can
do so at {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.create_account' %}.
If you would like to keep the information that requires consent available, please go to
{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.profile' %}, and review and
edit the information as desired. When ready, please check the 'Consent' checkbox found
at the bottom of the page and submit the form.
For information on how personal information is handled in the datatracker, please see
{{ settings.IDTRACKER_BASE_URL }}/help/personal-information.
Thank You,
The IETF Secretariat
{% endfilter %}

2
ietf/utils/log.py
View file

@ -43,7 +43,7 @@ def log(msg):
return
elif settings.DEBUG == True:
_logfunc = debug.say
_flushfunc = sys.stdout.flush
_flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition)
if isinstance(msg, unicode):
msg = msg.encode('unicode_escape')
try:

174
ietf/utils/management/commands/delete_data_lacking_consent.py Normal file
View file

@ -0,0 +1,174 @@
# Copyright The IETF Trust 2016, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function
import datetime
from tqdm import tqdm
from django.conf import settings
from django.contrib.admin.utils import NestedObjects
from django.contrib.auth.models import User
from django.core.management.base import BaseCommand
from django.db.models import F
import debug # pyflakes:ignore
from ietf.community.models import SearchRule
from ietf.person.models import Person, Alias, PersonalApiKey, Email
from ietf.person.name import unidecode_name
from ietf.utils.log import log
class Command(BaseCommand):
help = (u"""
Delete data for which consent to store the data has not been given,
where the data does not fall under the GDPR Legitimate Interest clause
for the IETF. This includes full name, ascii name, bio, login,
notification subscriptions and email addresses that are not derived from
published drafts or ietf roles.
""")
def add_arguments(self, parser):
parser.add_argument('-n', '--dry-run', action='store_true', default=False,
help="Don't delete anything, just list what would be done.")
# parser.add_argument('-d', '--date', help="Date of deletion (mentioned in message)")
parser.add_argument('-m', '--minimum-response-time', metavar='TIME', type=int, default=14,
help="Minimum response time, default: %(default)s days. Persons to whom a "
"consent request email has been sent more recently than this will not "
"be affected by the run.")
# parser.add_argument('-r', '--rate', type=float, default=1.0,
# help='Rate of sending mail, default: %(default)s/s')
# parser.add_argument('user', nargs='*')
def handle(self, *args, **options):
dry_run = options['dry_run']
verbosity = int(options['verbosity'])
event_type = 'gdpr_notice_email'
settings.DEBUG = False # don't log to console
# users
users = User.objects.filter(person__isnull=True, username__contains='@')
self.stdout.write("Found %d users without associated person records" % (users.count(), ))
emails = Email.objects.filter(address__in=users.values_list('username', flat=True))
# fix up users that don't have person records, but have a username matching a nown email record
self.stdout.write("Checking usernames against email records ...")
for email in tqdm(emails):
user = users.get(username=email.address)
if email.person.user_id:
if dry_run:
self.stdout.write("Would delete user #%-6s (%s) %s" % (user.id, user.last_login, user.username))
else:
log("Deleting user #%-6s (%s) %s: no person record, matching email has other user" % (user.id, user.last_login, user.username))
user_id = user.id
user.delete()
Person.history.filter(user_id=user_id).delete()
Email.history.filter(history_user=user_id).delete()
else:
if dry_run:
self.stdout.write("Would connect user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name()))
else:
log("Connecting user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name()))
email.person.user_id = user.id
email.person.save()
# delete users without person records
users = users.exclude(username__in=emails.values_list('address', flat=True))
if dry_run:
self.stdout.write("Would delete %d users without associated person records" % (users.count(), ))
else:
if users.count():
log("Deleting %d users without associated person records" % (users.count(), ))
assert not users.filter(person__isnull=False).exists()
user_ids = users.values_list('id', flat=True)
users.delete()
assert not Person.history.filter(user_id__in=user_ids).exists()
# persons
self.stdout.write('Querying the database for person records without given consent ...')
notification_cutoff = datetime.datetime.now() - datetime.timedelta(days=options['minimum_response_time'])
persons = Person.objects.exclude(consent=True)
persons = persons.exclude(id=1) # make sure we don't delete System ;-)
self.stdout.write("Found %d persons with information for which we don't have consent." % (persons.count(), ))
# Narrow to persons we don't have Legitimate Interest in, and delete those fully
persons = persons.exclude(docevent__by=F('pk'))
persons = persons.exclude(documentauthor__person=F('pk')).exclude(dochistoryauthor__person=F('pk'))
persons = persons.exclude(email__liaisonstatement__from_contact__person=F('pk'))
persons = persons.exclude(email__reviewrequest__reviewer__person=F('pk'))
persons = persons.exclude(email__shepherd_dochistory_set__shepherd__person=F('pk'))
persons = persons.exclude(email__shepherd_document_set__shepherd__person=F('pk'))
persons = persons.exclude(iprevent__by=F('pk'))
persons = persons.exclude(meetingregistration__person=F('pk'))
persons = persons.exclude(message__by=F('pk'))
persons = persons.exclude(name_from_draft='')
persons = persons.exclude(personevent__time__gt=notification_cutoff, personevent__type=event_type)
persons = persons.exclude(reviewrequest__requested_by=F('pk'))
persons = persons.exclude(role__person=F('pk')).exclude(rolehistory__person=F('pk'))
persons = persons.exclude(session__requested_by=F('pk'))
persons = persons.exclude(submissionevent__by=F('pk'))
self.stdout.write("Found %d persons with information for which we neither have consent nor legitimate interest." % (persons.count(), ))
if persons.count() > 0:
self.stdout.write("Deleting records for persons for which we have with neither consent nor legitimate interest ...")
for person in (persons if dry_run else tqdm(persons)):
if dry_run:
self.stdout.write(("Would delete record #%-6d: (%s) %-32s %-48s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email())).encode('utf8'))
else:
if verbosity > 1:
# development aids
collector = NestedObjects(using='default')
collector.collect([person,])
objects = collector.nested()
related = [ o for o in objects[-1] if not isinstance(o, (Alias, Person, SearchRule, PersonalApiKey)) ]
if len(related) > 0:
self.stderr.write("Person record #%-6s %s has unexpected related records" % (person.pk, person.ascii_name()))
# Historical records using simple_history has on_delete=DO_NOTHING, so
# we have to do explicit deletions:
id = person.id
person.delete()
Person.history.filter(id=id).delete()
Email.history.filter(person_id=id).delete()
# Deal with remaining persons (lacking consent, but with legitimate interest)
persons = Person.objects.exclude(consent=True)
persons = persons.exclude(id=1)
self.stdout.write("Found %d remaining persons with information for which we don't have consent." % (persons.count(), ))
if persons.count() > 0:
self.stdout.write("Removing personal information requiring consent ...")
for person in (persons if dry_run else tqdm(persons)):
fields = ', '.join(person.needs_consent())
if dry_run:
self.stdout.write(("Would remove info for #%-6d: (%s) %-32s %-48s %s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email(), fields)).encode('utf8'))
else:
if person.name_from_draft:
log("Using name info from draft for #%-6d %s: no consent, no roles" % (person.pk, person))
person.name = person.name_from_draft
person.ascii = unidecode_name(person.name_from_draft)
if person.biography:
log("Deleting biography for #%-6d %s: no consent, no roles" % (person.pk, person))
person.biography = ''
person.save()
if person.user_id:
if User.objects.filter(id=person.user_id).exists():
log("Deleting communitylist for #%-6d %s: no consent, no roles" % (person.pk, person))
person.user.communitylist_set.all().delete()
for email in person.email_set.all():
if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]:
log("Deleting email <%s> for #%-6d %s: no consent, no roles" % (email.address, person.pk, person))
address = email.address
email.delete()
Email.history.filter(address=address).delete()
emails = Email.objects.filter(origin='', person__consent=False)
self.stdout.write("Found %d emails without origin for which we lack consent." % (emails.count(), ))
if dry_run:
self.stdout.write("Would delete %d email records without origin and consent" % (emails.count(), ))
else:
if emails.count():
log("Deleting %d email records without origin and consent" % (emails.count(), ))
addresses = emails.values_list('address', flat=True)
emails.delete()
Email.history.filter(address__in=addresses).delete()

92
ietf/utils/management/commands/send_gdpr_consent_request.py Normal file
View file

@ -0,0 +1,92 @@
# Copyright The IETF Trust 2016, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function
import datetime
import time
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import debug # pyflakes:ignore
from ietf.person.models import Person, PersonEvent
from ietf.utils.mail import send_mail
class Command(BaseCommand):
help = (u"""
Send GDPR consent request emails to persons who have not indicated consent
to having their personal information stored. Each send is logged as a
PersonEvent.
By default email sending happens at a rate of 1 message per second; the
rate can be adjusted with the -r option. At the start of a run, an estimate
is given of how many persons to send to, and how long the run will take.
By default, emails are not sent out if there is less than 6 days since the
previous consent request email. The interval can be adjusted with the -m
option. One effect of this is that it is possible to break of a run and
re-start it with for instance a different rate, without having duplicate
messages go out to persons that were handled in the interrupted run.
""")
def add_arguments(self, parser):
parser.add_argument('-n', '--dry-run', action='store_true', default=False,
help="Don't send email, just list recipients")
parser.add_argument('-d', '--date', help="Date of deletion (mentioned in message)")
parser.add_argument('-m', '--minimum-interval', type=int, default=6,
help="Minimum interval between re-sending email messages, default: %(default)s days")
parser.add_argument('-r', '--rate', type=float, default=1.0,
help='Rate of sending mail, default: %(default)s/s')
parser.add_argument('user', nargs='*')
def handle(self, *args, **options):
event_type = 'gdpr_notice_email'
# Arguments
# --date
if 'date' in options and options['date'] != None:
try:
date = datetime.datetime.strptime(options['date'], "%Y-%m-%d").date()
except ValueError as e:
raise CommandError('%s' % e)
else:
date = datetime.date.today() + datetime.timedelta(days=30)
days = (date - datetime.date.today()).days
if days <= 1:
raise CommandError('date must be more than 1 day in the future')
# --rate
delay = 1.0/options['rate']
# --minimum_interval
minimum_interval = options['minimum_interval']
latest_previous = datetime.datetime.now() - datetime.timedelta(days=minimum_interval)
# user
self.stdout.write('Querying the database for matching person records ...')
if 'user' in options and options['user']:
persons = Person.objects.filter(user__username__in=options['user'])
else:
persons = Person.objects.exclude(consent=True).exclude(personevent__time__gt=latest_previous, personevent__type=event_type)
# Report the size of the run
runtime = persons.count() * delay
self.stdout.write('Sending to %d users; estimated time a bit more than %d:%02d hours' % (persons.count(), runtime//3600, runtime%3600//60))
for person in persons:
fields = ', '.join(person.needs_consent())
if fields and person.email_set.exists():
if options['dry_run']:
print(("%-32s %-32s %-32s %-32s %s" % (person.email(), person.name_from_draft or '', person.name, person.ascii, fields)).encode('utf8'))
else:
to = [ e.address for e in person.email_set.filter(active=True) ] # pyflakes:ignore
if not to:
to = [ e.address for e in person.email_set.all() ] # pyflakes:ignore
self.stdout.write("Sendimg email to %s" % to)
send_mail(None, to, None,
subject='Personal Information in the IETF Datatracker',
template='utils/personal_information_notice.txt',
context={
'date': date, 'days': days, 'fields': fields,
'person': person, 'settings': settings,
},
)
e = PersonEvent.objects.create(person=person, type='gdpr_notice_email',
desc="Sent GDPR notice email to %s with confirmation deadline %s" % (to, date))
time.sleep(delay)

BIN
release-coverage.json.gz
View file

Binary file not shown.