From 35e16ef7d5c85c280107a520b16e3e13110b4369 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 14 Jun 2018 18:53:48 +0000 Subject: [PATCH 01/12] Changed User ForeignKeys to not delete objects pointing at a user when the user is deleted, instead setting the ForeignKey field to None. - Legacy-Id: 15257 --- ietf/nomcom/models.py | 4 ++-- ietf/person/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py index 91c946d3a..8b1407ad5 100644 --- a/ietf/nomcom/models.py +++ b/ietf/nomcom/models.py @@ -90,7 +90,7 @@ class Nomination(models.Model): nominee = ForeignKey('Nominee') comments = ForeignKey('Feedback') nominator_email = models.EmailField(verbose_name='Nominator Email', blank=True) - user = ForeignKey(User, editable=False) + user = ForeignKey(User, editable=False, null=True, on_delete=models.SET_NULL) time = models.DateTimeField(auto_now_add=True) share_nominator = models.BooleanField(verbose_name='Share nominator name with candidate', default=False, help_text='Check this box to allow the NomCom to let the ' @@ -247,7 +247,7 @@ class Feedback(models.Model): subject = models.TextField(verbose_name='Subject', blank=True) comments = EncryptedTextField(verbose_name='Comments') type = ForeignKey(FeedbackTypeName, blank=True, null=True) - user = ForeignKey(User, editable=False, blank=True, null=True) + user = ForeignKey(User, editable=False, blank=True, null=True, on_delete=models.SET_NULL) time = models.DateTimeField(auto_now_add=True) objects = FeedbackManager() diff --git a/ietf/person/models.py b/ietf/person/models.py index 40b80c3f6..23753bcff 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -31,7 +31,7 @@ from ietf.utils.models import ForeignKey, OneToOneField class Person(models.Model): history = HistoricalRecords() - user = OneToOneField(User, blank=True, null=True) + user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL) time = models.DateTimeField(default=datetime.datetime.now) # When this Person record entered the system # The normal unicode form of the name. This must be # set to the same value as the ascii-form if equal. From 31352b00645882a766c5fa7dbc30c6eaa92774b2 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 14 Jun 2018 18:56:01 +0000 Subject: [PATCH 02/12] Added Person.needs_consent() which returns a list of descriptions of fields for which consent is needed, or []. - Legacy-Id: 15258 --- ietf/person/models.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ietf/person/models.py b/ietf/person/models.py index 23753bcff..bf932c360 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -163,6 +163,25 @@ class Person(models.Model): from ietf.doc.models import Document return Document.objects.filter(documentauthor__person=self, type='draft', states__slug__in=['repl', 'expired', 'auth-rm', 'ietf-rm']).order_by('-time') + def needs_consent(self): + """ + Returns an empty list or a list of fields which holds information that + requires consent to be given. + """ + needs_consent = [] + if self.name != self.name_from_draft: + needs_consent.append("full name") + if self.ascii != self.name_from_draft: + needs_consent.append("ascii name") + if self.biography and self.role_set.count(): + needs_consent.append("biography") + if self.user and self.user.communitylist_set.exists(): + needs_consent.append("draft notification subscription(s)") + for email in self.email_set.all(): + if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]: + needs_consent.append("email address(es)") + return needs_consent + def save(self, *args, **kwargs): created = not self.pk super(Person, self).save(*args, **kwargs) From 50632482ea41a2644f7917b4b3a89e9bb78cc104 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 14 Jun 2018 19:27:03 +0000 Subject: [PATCH 03/12] Improved the wording of the consent required login warnin. Refactored the login view to use Person.needs_consent() instead of inline logic. - Legacy-Id: 15260 --- ietf/ietfauth/views.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 07b4debbc..c2e37c5d2 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -580,25 +580,14 @@ def login(request, extra_context=None): which is not recognized as a valid password hash. """ + require_consent = [] if request.method == "POST": form = AuthenticationForm(request, data=request.POST) username = form.data.get('username') user = User.objects.filter(username=username).first() # - require_consent = [] if user.person and not user.person.consent: - person = user.person - if person.name != person.name_from_draft: - require_consent.append("full name") - if person.ascii != person.name_from_draft: - require_consent.append("ascii name") - if person.biography: - require_consent.append("biography") - if user.communitylist_set.exists(): - require_consent.append("draft notification subscription(s)") - for email in person.email_set.all(): - if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]: - require_consent.append("email address(es)") + require_consent = user.person.needs_consent() if user: try: identify_hasher(user.password) @@ -617,8 +606,10 @@ def login(request, extra_context=None): You have personal information associated with your account which is not derived from draft submissions or other ietf work, namely: %s. Please go to your account profile and review your - personal information, and confirm that it may be used and displayed - within the IETF datatracker. + personal information, then scoll to the bottom and check the 'confirm' + checkbox and submit the form, in order to to indicate that that the + provided personal information may be used and displayed within the IETF + datatracker. """ % ', '.join(require_consent))) return response From 5251d0eb787ab340fe4ca2b1d46b006737d87b44 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 14 Jun 2018 19:28:06 +0000 Subject: [PATCH 04/12] Changed the handling of the consent field of /account/profile/ to only be required set when information requiring consent is present. - Legacy-Id: 15261 --- ietf/ietfauth/forms.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py index 5349b83c0..ab42fd144 100644 --- a/ietf/ietfauth/forms.py +++ b/ietf/ietfauth/forms.py @@ -116,8 +116,6 @@ def get_person_form(*args, **kwargs): if f in self.fields: self.fields[f].label += ' \u2020' - self.fields["consent"].required = True - self.unidecoded_ascii = False if self.data and not self.data.get("ascii", "").strip(): @@ -150,8 +148,13 @@ def get_person_form(*args, **kwargs): def clean_consent(self): consent = self.cleaned_data.get('consent') - if consent == False: - raise forms.ValidationError("In order to modify your profile data, you must permit the IETF to use the uploaded data.") + require_consent = ( + self.cleaned_data.get('name') != person.name_from_draft + or self.cleaned_data.get('ascii') != person.name_from_draft + or self.cleaned_data.get('biography') + ) + if consent == False and require_consent: + raise forms.ValidationError("In order to modify your profile with data that require consent, you must permit the IETF to use the uploaded data.") return consent return PersonForm(*args, **kwargs) From d8005ab0f187559ff5901ea33ee9903f9d12540c Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Tue, 19 Jun 2018 19:19:58 +0000 Subject: [PATCH 05/12] Fixed an issue in Person.needs_consent(): avoid multiple mentions of 'email address'. - Legacy-Id: 15268 --- ietf/person/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/person/models.py b/ietf/person/models.py index bf932c360..b30ef4d04 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -180,6 +180,7 @@ class Person(models.Model): for email in self.email_set.all(): if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]: needs_consent.append("email address(es)") + break return needs_consent def save(self, *args, **kwargs): From 97db5f71b55e61c0634ecf4847e83f7add9e258a Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Tue, 19 Jun 2018 19:23:46 +0000 Subject: [PATCH 06/12] Added a management command to send out gdpr consent requests. - Legacy-Id: 15269 --- .../utils/personal_information_notice.txt | 20 +++++++++ .../commands/send_gdpr_consent_request.py | 41 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 ietf/templates/utils/personal_information_notice.txt create mode 100644 ietf/utils/management/commands/send_gdpr_consent_request.py diff --git a/ietf/templates/utils/personal_information_notice.txt b/ietf/templates/utils/personal_information_notice.txt new file mode 100644 index 000000000..8f4d5d2f8 --- /dev/null +++ b/ietf/templates/utils/personal_information_notice.txt @@ -0,0 +1,20 @@ +{% load ietf_filters %}{% filter wordwrap:78 %} +Dear {{ person.plain_name }}, + +This email concerns some personal information stored in your IETF datatracker profile that +requires your consent for storage and use. + +If you do nothing in response to this email, the information in your profile that requires +consent ({{ fields|safe }}) will be deleted one month from now, on {{ date }}. + +If you would like to keep the information that requires consent available, please go to +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.profile' %}, check the +information, scroll down to the end of the page, check the 'Consent' checkbox and submit +the form. + +For information on how personal information is handled in the datatracker, please see +{{ settings.IDTRACKER_BASE_URL }}/help/personal-information. + +Thank You, +The IETF Secretariat +{% endfilter %} diff --git a/ietf/utils/management/commands/send_gdpr_consent_request.py b/ietf/utils/management/commands/send_gdpr_consent_request.py new file mode 100644 index 000000000..cd42fe0db --- /dev/null +++ b/ietf/utils/management/commands/send_gdpr_consent_request.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2016, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + +import syslog +import datetime + +from django.conf import settings +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.person.models import Person +from ietf.utils.mail import send_mail + +def log(message): + syslog.syslog(message) + +class Command(BaseCommand): + help = (u"Send GDPR consent requests to those that need it") + + def add_arguments(self, parser): + parser.add_argument('-n', '--dry-run', dest='dryrun', action='store_true', default=False, + help="Don't send email, just list recipients") + + def handle(self, *args, **options): + for person in Person.objects.exclude(consent=True): + fields = ', '.join(person.needs_consent()) + date = datetime.date.today() + datetime.timedelta(days=30) + if fields and person.email_set.exists(): + if options['dryrun']: + print(("%-32s %-32s %-32s %-32s %s" % (person.email(), person.name_from_draft or '', person.name, person.ascii, fields)).encode('utf8')) + else: + to = [ e.address for e in person.email_set.filter(active=True) ] + if not to: + to = [ e.address for e in person.email_set.all() ] + send_mail(None, to, None, + subject='Personal Information in the IETF Datatracker', + template='utils/personal_information_notice.txt', + context={'fields': fields, 'person': person, 'settings': settings, 'date': date, }, ) + From 74359e953876e55d4d252aecab89c6786b76e15f Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:44:27 +0000 Subject: [PATCH 07/12] Refined Person.plain_name() to avoid leading or trailing spaces when only one name component is present. - Legacy-Id: 15456 --- ietf/person/name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/person/name.py b/ietf/person/name.py index 26607d616..e981df591 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -71,7 +71,7 @@ def initials(name): def plain_name(name): prefix, first, middle, last, suffix = name_parts(name) - return u" ".join([first, last]) + return u" ".join( n for n in (first, last) if n) def capfirst(s): # Capitalize the first word character, skipping non-word characters and From 44d4d760924f54431ddc9c59416004520c2726dd Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:46:28 +0000 Subject: [PATCH 08/12] Added a migration to capture a change in on_delete for the Person.user field. - Legacy-Id: 15457 --- .../migrations/0006_auto_20180910_0719.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ietf/person/migrations/0006_auto_20180910_0719.py diff --git a/ietf/person/migrations/0006_auto_20180910_0719.py b/ietf/person/migrations/0006_auto_20180910_0719.py new file mode 100644 index 000000000..cfdb07b6e --- /dev/null +++ b/ietf/person/migrations/0006_auto_20180910_0719.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.13 on 2018-09-10 07:19 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0005_populate_person_name_from_draft'), + ] + + operations = [ + migrations.AlterField( + model_name='person', + name='user', + field=ietf.utils.models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + ] From 2aefd51083887fb5ca1803e34c57c32371e49989 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:47:08 +0000 Subject: [PATCH 09/12] Updated the personal information notification email template. - Legacy-Id: 15458 --- .../utils/personal_information_notice.txt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ietf/templates/utils/personal_information_notice.txt b/ietf/templates/utils/personal_information_notice.txt index 8f4d5d2f8..9a3fd692e 100644 --- a/ietf/templates/utils/personal_information_notice.txt +++ b/ietf/templates/utils/personal_information_notice.txt @@ -1,20 +1,23 @@ {% load ietf_filters %}{% filter wordwrap:78 %} Dear {{ person.plain_name }}, -This email concerns some personal information stored in your IETF datatracker profile that -requires your consent for storage and use. +This email is regarding some of the personal information stored in your IETF datatracker +profile; information for which we require your consent for storage and use. -If you do nothing in response to this email, the information in your profile that requires -consent ({{ fields|safe }}) will be deleted one month from now, on {{ date }}. +If you do nothing in response to this email, the information in your profile +that requires consent ({{ fields|safe }} and login) will be deleted {{ days }} +days from now, on {{ date }}. If you later wish to create a new login, you can +do so at {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.create_account' %}. If you would like to keep the information that requires consent available, please go to -{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.profile' %}, check the -information, scroll down to the end of the page, check the 'Consent' checkbox and submit -the form. +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.ietfauth.views.profile' %}, and review and +edit the information as desired. When ready, please check the 'Consent' checkbox found +at the bottom of the page and submit the form. For information on how personal information is handled in the datatracker, please see {{ settings.IDTRACKER_BASE_URL }}/help/personal-information. + Thank You, The IETF Secretariat {% endfilter %} From 75f703578cea8bbb49fbfd8deb1e1265c953d39f Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:49:25 +0000 Subject: [PATCH 10/12] Tweaked the Person.needs_consent() method to take historic roles into account and to handle records without associated usesrs. - Legacy-Id: 15459 --- ietf/person/models.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ietf/person/models.py b/ietf/person/models.py index b30ef4d04..b62918901 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -12,6 +12,7 @@ from urlparse import urljoin from django.conf import settings from django.core.validators import validate_email +from django.core.exceptions import ObjectDoesNotExist from django.db import models from django.contrib.auth.models import User from django.template.loader import render_to_string @@ -173,10 +174,15 @@ class Person(models.Model): needs_consent.append("full name") if self.ascii != self.name_from_draft: needs_consent.append("ascii name") - if self.biography and self.role_set.count(): + if self.biography and not (self.role_set.exists() or self.rolehistory_set.exists()): needs_consent.append("biography") - if self.user and self.user.communitylist_set.exists(): - needs_consent.append("draft notification subscription(s)") + if self.user_id: + needs_consent.append("login") + try: + if self.user.communitylist_set.exists(): + needs_consent.append("draft notification subscription(s)") + except ObjectDoesNotExist: + pass for email in self.email_set.all(): if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]: needs_consent.append("email address(es)") From 3f9dbeada8a22adf60d39db03f7ab514bae7b491 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:52:29 +0000 Subject: [PATCH 11/12] Expanded the send_gdpr_consent_request command to add support for deletion dates, email reminder interval, email sending rate limiting, and sending to a list of users. - Legacy-Id: 15460 --- .../commands/send_gdpr_consent_request.py | 79 ++++++++++++++++--- 1 file changed, 66 insertions(+), 13 deletions(-) diff --git a/ietf/utils/management/commands/send_gdpr_consent_request.py b/ietf/utils/management/commands/send_gdpr_consent_request.py index cd42fe0db..9f23e7379 100644 --- a/ietf/utils/management/commands/send_gdpr_consent_request.py +++ b/ietf/utils/management/commands/send_gdpr_consent_request.py @@ -2,40 +2,93 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals, print_function -import syslog import datetime +import sys +import time from django.conf import settings -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError import debug # pyflakes:ignore -from ietf.person.models import Person +from ietf.person.models import Person, PersonEvent from ietf.utils.mail import send_mail - -def log(message): - syslog.syslog(message) +from ietf.utils.log import log class Command(BaseCommand): - help = (u"Send GDPR consent requests to those that need it") + help = (u""" + Send GDPR consent request emails to persons who have not indicated consent + to having their personal information stored. Each send is logged as a + PersonEvent. + + By default email sending happens at a rate of 1 message per second; the + rate can be adjusted with the -r option. At the start of a run, an estimate + is given of how many persons to send to, and how long the run will take. + + By default, emails are not sent out if there is less than 6 days since the + previous consent request email. The interval can be adjusted with the -m + option. One effect of this is that it is possible to break of a run and + re-start it with for instance a different rate, without having duplicate + messages go out to persons that were handled in the interrupted run. + """) def add_arguments(self, parser): - parser.add_argument('-n', '--dry-run', dest='dryrun', action='store_true', default=False, + parser.add_argument('-n', '--dry-run', action='store_true', default=False, help="Don't send email, just list recipients") + parser.add_argument('-d', '--date', help="Date of deletion (mentioned in message)") + parser.add_argument('-m', '--minimum-interval', type=int, default=6, + help="Minimum interval between re-sending email messages, default: %(default)s days") + parser.add_argument('-r', '--rate', type=float, default=1.0, + help='Rate of sending mail, default: %(default)s/s') + parser.add_argument('user', nargs='*') + def handle(self, *args, **options): - for person in Person.objects.exclude(consent=True): - fields = ', '.join(person.needs_consent()) + event_type = 'gdpr_notice_email' + # Arguments + # --date + if 'date' in options and options['date'] != None: + try: + date = datetime.datetime.strptime(options['date'], "%Y-%m-%d").date() + except ValueError as e: + raise CommandError('%s' % e) + else: date = datetime.date.today() + datetime.timedelta(days=30) + days = (date - datetime.date.today()).days + if days <= 1: + raise CommandError('date must be more than 1 day in the future') + # --rate + delay = 1.0/options['rate'] + # --minimum_interval + minimum_interval = options['minimum_interval'] + latest_previous = datetime.datetime.now() - datetime.timedelta(days=minimum_interval) + # user + self.stdout.write('Querying the database for matching person records ...') + if 'user' in options and options['user']: + persons = Person.objects.filter(user__username__in=options['user']) + else: + persons = Person.objects.exclude(consent=True).exclude(personevent__time__gt=latest_previous, personevent__type=event_type) + # Report the size of the run + runtime = persons.count() * delay + self.stdout.write('Sending to %d users; estimated time a bit more than %d:%02d hours' % (persons.count(), runtime//3600, runtime%3600//60)) + for person in persons: + fields = ', '.join(person.needs_consent()) if fields and person.email_set.exists(): - if options['dryrun']: + if options['dry_run']: print(("%-32s %-32s %-32s %-32s %s" % (person.email(), person.name_from_draft or '', person.name, person.ascii, fields)).encode('utf8')) else: to = [ e.address for e in person.email_set.filter(active=True) ] if not to: to = [ e.address for e in person.email_set.all() ] + self.stdout.write("Sendimg email to %s" % to) send_mail(None, to, None, subject='Personal Information in the IETF Datatracker', template='utils/personal_information_notice.txt', - context={'fields': fields, 'person': person, 'settings': settings, 'date': date, }, ) - + context={ + 'date': date, 'days': days, 'fields': fields, + 'person': person, 'settings': settings, + }, + ) + e = PersonEvent.objects.create(person=person, type='gdpr_notice_email', + desc="Sent GDPR notice email to %s with confirmation deadline %s" % (to, date)) + time.sleep(delay) From b85e1c46d9da63185bbdc83e42f85937f1ce540c Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 16 Sep 2018 13:54:32 +0000 Subject: [PATCH 12/12] Added a delete_data_lacking_consent management command that deletes person records and person information for which we need consent according to GDPR, but have not received it. - Legacy-Id: 15461 --- .../commands/delete_data_lacking_consent.py | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 ietf/utils/management/commands/delete_data_lacking_consent.py diff --git a/ietf/utils/management/commands/delete_data_lacking_consent.py b/ietf/utils/management/commands/delete_data_lacking_consent.py new file mode 100644 index 000000000..e259d5443 --- /dev/null +++ b/ietf/utils/management/commands/delete_data_lacking_consent.py @@ -0,0 +1,180 @@ +# Copyright The IETF Trust 2016, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, print_function + +import datetime +import sys +import time +from tqdm import tqdm + +from django.conf import settings +from django.contrib.admin.utils import NestedObjects +from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand, CommandError +from django.db.models import F + +import debug # pyflakes:ignore + +from ietf.community.models import SearchRule +from ietf.nomcom.models import Feedback, Nomination +from ietf.person.models import Person, Alias, PersonEvent, PersonalApiKey, Email +from ietf.person.name import unidecode_name +from ietf.stats.models import MeetingRegistration +from ietf.utils.mail import send_mail +from ietf.utils.log import log + +class Command(BaseCommand): + help = (u""" + + Delete data for which consent to store the data has not been given, + where the data does not fall under the GDPR Legitimate Interest clause + for the IETF. This includes full name, ascii name, bio, login, + notification subscriptions and email addresses that are not derived from + published drafts or ietf roles. + + """) + + def add_arguments(self, parser): + parser.add_argument('-n', '--dry-run', action='store_true', default=False, + help="Don't delete anything, just list what would be done.") +# parser.add_argument('-d', '--date', help="Date of deletion (mentioned in message)") + parser.add_argument('-m', '--minimum-response-time', metavar='TIME', type=int, default=14, + help="Minimum response time, default: %(default)s days. Persons to whom a " + "consent request email has been sent more recently than this will not " + "be affected by the run.") +# parser.add_argument('-r', '--rate', type=float, default=1.0, +# help='Rate of sending mail, default: %(default)s/s') +# parser.add_argument('user', nargs='*') + + + def handle(self, *args, **options): + dry_run = options['dry_run'] + verbosity = int(options['verbosity']) + event_type = 'gdpr_notice_email' + settings.DEBUG = False # don't log to console + + # users + users = User.objects.filter(person__isnull=True, username__contains='@') + self.stdout.write("Found %d users without associated person records" % (users.count(), )) + emails = Email.objects.filter(address__in=users.values_list('username', flat=True)) + # fix up users that don't have person records, but have a username matching a nown email record + self.stdout.write("Checking usernames against email records ...") + for email in tqdm(emails): + user = users.get(username=email.address) + if email.person.user_id: + if dry_run: + self.stdout.write("Would delete user #%-6s (%s) %s" % (user.id, user.last_login, user.username)) + else: + log("Deleting user #%-6s (%s) %s: no person record, matching email has other user" % (user.id, user.last_login, user.username)) + user_id = user.id + user.delete() + Person.history.filter(user_id=user_id).delete() + Email.history.filter(history_user=user_id).delete() + else: + if dry_run: + self.stdout.write("Would connect user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name())) + else: + log("Connecting user #%-6s %s to person #%-6s %s" % (user.id, user.username, email.person.id, email.person.ascii_name())) + email.person.user_id = user.id + email.person.save() + # delete users without person records + users = users.exclude(username__in=emails.values_list('address', flat=True)) + if dry_run: + self.stdout.write("Would delete %d users without associated person records" % (users.count(), )) + else: + if users.count(): + log("Deleting %d users without associated person records" % (users.count(), )) + assert not users.filter(person__isnull=False).exists() + user_ids = users.values_list('id', flat=True) + users.delete() + assert not Person.history.filter(user_id__in=user_ids).exists() + + + # persons + self.stdout.write('Querying the database for person records without given consent ...') + notification_cutoff = datetime.datetime.now() - datetime.timedelta(days=options['minimum_response_time']) + persons = Person.objects.exclude(consent=True) + persons = persons.exclude(id=1) # make sure we don't delete System ;-) + self.stdout.write("Found %d persons with information for which we don't have consent." % (persons.count(), )) + + # Narrow to persons we don't have Legitimate Interest in, and delete those fully + persons = persons.exclude(docevent__by=F('pk')) + persons = persons.exclude(documentauthor__person=F('pk')).exclude(dochistoryauthor__person=F('pk')) + persons = persons.exclude(email__liaisonstatement__from_contact__person=F('pk')) + persons = persons.exclude(email__reviewrequest__reviewer__person=F('pk')) + persons = persons.exclude(email__shepherd_dochistory_set__shepherd__person=F('pk')) + persons = persons.exclude(email__shepherd_document_set__shepherd__person=F('pk')) + persons = persons.exclude(iprevent__by=F('pk')) + persons = persons.exclude(meetingregistration__person=F('pk')) + persons = persons.exclude(message__by=F('pk')) + persons = persons.exclude(name_from_draft='') + persons = persons.exclude(personevent__time__gt=notification_cutoff, personevent__type=event_type) + persons = persons.exclude(reviewrequest__requested_by=F('pk')) + persons = persons.exclude(role__person=F('pk')).exclude(rolehistory__person=F('pk')) + persons = persons.exclude(session__requested_by=F('pk')) + persons = persons.exclude(submissionevent__by=F('pk')) + self.stdout.write("Found %d persons with information for which we neither have consent nor legitimate interest." % (persons.count(), )) + if persons.count() > 0: + self.stdout.write("Deleting records for persons for which we have with neither consent nor legitimate interest ...") + for person in (persons if dry_run else tqdm(persons)): + if dry_run: + self.stdout.write(("Would delete record #%-6d: (%s) %-32s %-48s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email())).encode('utf8')) + else: + if verbosity > 1: + # development aids + collector = NestedObjects(using='default') + collector.collect([person,]) + objects = collector.nested() + related = [ o for o in objects[-1] if not isinstance(o, (Alias, Person, SearchRule, PersonalApiKey)) ] + if len(related) > 0: + self.stderr.write("Person record #%-6s %s has unexpected related records" % (person.pk, person.ascii_name())) + + # Historical records using simple_history has on_delete=DO_NOTHING, so + # we have to do explicit deletions: + id = person.id + person.delete() + Person.history.filter(id=id).delete() + Email.history.filter(person_id=id).delete() + + # Deal with remaining persons (lacking consent, but with legitimate interest) + persons = Person.objects.exclude(consent=True) + persons = persons.exclude(id=1) + self.stdout.write("Found %d remaining persons with information for which we don't have consent." % (persons.count(), )) + if persons.count() > 0: + self.stdout.write("Removing personal information requiring consent ...") + for person in (persons if dry_run else tqdm(persons)): + fields = ', '.join(person.needs_consent()) + if dry_run: + self.stdout.write(("Would remove info for #%-6d: (%s) %-32s %-48s %s" % (person.pk, person.time, person.ascii_name(), "<%s>"%person.email(), fields)).encode('utf8')) + else: + if person.name_from_draft: + log("Using name info from draft for #%-6d %s: no consent, no roles" % (person.pk, person)) + person.name = person.name_from_draft + person.ascii = unidecode_name(person.name_from_draft) + if person.biography: + log("Deleting biography for #%-6d %s: no consent, no roles" % (person.pk, person)) + person.biography = '' + person.save() + if person.user_id: + if User.objects.filter(id=person.user_id).exists(): + log("Deleting communitylist for #%-6d %s: no consent, no roles" % (person.pk, person)) + person.user.communitylist_set.all().delete() + for email in person.email_set.all(): + if not email.origin.split(':')[0] in ['author', 'role', 'reviewer', 'liaison', 'shepherd', ]: + log("Deleting email <%s> for #%-6d %s: no consent, no roles" % (email.address, person.pk, person)) + address = email.address + email.delete() + Email.history.filter(address=address).delete() + + emails = Email.objects.filter(origin='', person__consent=False) + self.stdout.write("Found %d emails without origin for which we lack consent." % (emails.count(), )) + if dry_run: + self.stdout.write("Would delete %d email records without origin and consent" % (emails.count(), )) + else: + if emails.count(): + log("Deleting %d email records without origin and consent" % (emails.count(), )) + addresses = emails.values_list('address', flat=True) + emails.delete() + Email.history.filter(address__in=addresses).delete() +