From 152261a869967e2453133f286ede3bdfc2be7241 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Thu, 14 Dec 2017 14:30:59 +0000 Subject: [PATCH 1/6] Added new infrastructure for personal API keys, to generate, view, and delete them. - Legacy-Id: 14423 --- ietf/ietfauth/tests.py | 58 +++++++++++++++++++- ietf/ietfauth/urls.py | 3 ++ ietf/ietfauth/views.py | 79 +++++++++++++++++++++++----- ietf/person/admin.py | 9 +++- ietf/person/models.py | 48 +++++++++++++++++ ietf/person/resources.py | 22 +++++++- ietf/templates/base.html | 2 +- ietf/templates/base/menu_user.html | 1 + ietf/templates/form.html | 34 ++++++++++++ ietf/templates/ietfauth/apikeys.html | 47 +++++++++++++++++ ietf/templates/person/profile.html | 10 ++-- ietf/utils/decorators.py | 52 ++++++++++++++++++ 12 files changed, 342 insertions(+), 23 deletions(-) create mode 100644 ietf/templates/form.html create mode 100644 ietf/templates/ietfauth/apikeys.html diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index abf3ec027..f261c3075 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -16,10 +16,11 @@ import debug # pyflakes:ignore from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.test_data import make_test_data, make_review_data from ietf.utils.mail import outbox, empty_outbox -from ietf.person.models import Person, Email from ietf.group.models import Group, Role, RoleName from ietf.ietfauth.htpasswd import update_htpasswd_file from ietf.mailinglists.models import Subscribed +from ietf.person.models import Person, Email +from ietf.person.factories import PersonFactory from ietf.review.models import ReviewWish, UnavailablePeriod from ietf.utils.decorators import skip_coverage @@ -495,3 +496,58 @@ class IetfAuthTests(TestCase): user = User.objects.get(username="othername@example.org") self.assertEqual(prev, user) self.assertTrue(user.check_password(u'password')) + + def test_apikey(self): + person = PersonFactory() + + url = urlreverse('ietf.ietfauth.views.apikey_index') + + # Check that the url is protected, then log in + login_testing_unauthorized(self, person.user.username, url) + + # Check api key list content + r = self.client.get(url) + self.assertContains(r, 'Personal API keys') + self.assertContains(r, 'Get a new personal API key') + + # Check the add key form content + url = urlreverse('ietf.ietfauth.views.apikey_add') + r = self.client.get(url) + self.assertContains(r, 'Create a new personal API key') + self.assertContains(r, 'Endpoint') + + # Add 2 keys + r = self.client.post(url, {'endpoint': '/api/submit'}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + r = self.client.post(url, {'endpoint': '/api/iesg/discuss'}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + + # Check api key list content + url = urlreverse('ietf.ietfauth.views.apikey_index') + r = self.client.get(url) + self.assertContains(r, '/api/submit') + self.assertContains(r, '/api/iesg/discuss') + q = PyQuery(r.content) + self.assertEqual(len(q('td code')), 2) + + # Get one of the keys + key = person.apikeys.first() + + # Check the delete key form content + url = urlreverse('ietf.ietfauth.views.apikey_del') + r = self.client.get(url) + + self.assertContains(r, 'Delete a personal API key') + self.assertContains(r, 'Key') + + # Delete a key + r = self.client.post(url, {'hash': key.hash()}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + + # Check the api key list content again + url = urlreverse('ietf.ietfauth.views.apikey_index') + r = self.client.get(url) + self.assertNotContains(r, key.endpoint) + q = PyQuery(r.content) + self.assertEqual(len(q('td code')), 1) + diff --git a/ietf/ietfauth/urls.py b/ietf/ietfauth/urls.py index 5efbb4b38..b586d284b 100644 --- a/ietf/ietfauth/urls.py +++ b/ietf/ietfauth/urls.py @@ -7,6 +7,9 @@ from ietf.utils.urls import url urlpatterns = [ url(r'^$', views.index), + url(r'^apikey/?$', views.apikey_index), + url(r'^apikey/add/?$', views.apikey_add), + url(r'^apikey/del/?$', views.apikey_del), url(r'^confirmnewemail/(?P[^/]+)/$', views.confirm_new_email), url(r'^create/$', views.create_account), url(r'^create/confirm/(?P[^/]+)/$', views.confirm_account), diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 8739766ff..99f4249bd 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -48,6 +48,7 @@ from django.contrib.auth.hashers import identify_hasher from django.contrib.auth.models import User from django.contrib.auth.views import login as django_login from django.contrib.sites.models import Site +from django.core.validators import ValidationError from django.urls import reverse as urlreverse from django.http import Http404, HttpResponseRedirect #, HttpResponse, from django.shortcuts import render, redirect, get_object_or_404 @@ -61,11 +62,12 @@ from ietf.ietfauth.forms import ( RegistrationForm, PasswordForm, ResetPasswordF from ietf.ietfauth.htpasswd import update_htpasswd_file from ietf.ietfauth.utils import role_required from ietf.mailinglists.models import Subscribed, Whitelisted -from ietf.person.models import Person, Email, Alias +from ietf.person.models import Person, Email, Alias, PersonalApiKey from ietf.review.models import ReviewRequest, ReviewerSettings, ReviewWish from ietf.review.utils import unavailable_periods_to_list, get_default_filter_re -from ietf.utils.mail import send_mail from ietf.doc.fields import SearchableDocumentField +from ietf.utils.decorators import person_required +from ietf.utils.mail import send_mail def index(request): return render(request, 'registration/index.html') @@ -190,14 +192,10 @@ def confirm_account(request, auth): }) @login_required +@person_required def profile(request): roles = [] - person = None - - try: - person = request.user.person - except Person.DoesNotExist: - return render(request, 'registration/missing_person.html') + person = request.user.person roles = Role.objects.filter(person=person, group__state='active').order_by('name__name', 'group__name') emails = Email.objects.filter(person=person).order_by('-active','-time') @@ -533,13 +531,9 @@ def change_password(request): @login_required +@person_required def change_username(request): - person = None - - try: - person = request.user.person - except Person.DoesNotExist: - return render(request, 'registration/missing_person.html') + person = request.user.person emails = [ e.address for e in Email.objects.filter(person=person, active=True) ] emailz = [ e.address for e in person.email_set.filter(active=True) ] @@ -599,3 +593,60 @@ def login(request, extra_context=None): } return django_login(request, extra_context=extra_context) + +@login_required +@person_required +def apikey_index(request): + person = request.user.person + return render(request, 'ietfauth/apikeys.html', {'person': person}) + +@login_required +@person_required +def apikey_add(request): + class ApiKeyForm(forms.ModelForm): + class Meta: + model = PersonalApiKey + fields = ['endpoint'] + # + person = request.user.person + if request.method == 'POST': + form = ApiKeyForm(request.POST) + if form.is_valid(): + api_key = form.save(commit=False) + api_key.person = person + api_key.save() + return redirect('ietf.ietfauth.views.apikey_index') + else: + form = ApiKeyForm() + return render(request, 'form.html', {'form':form, 'title':"Create a new personal API key", 'description':'', 'button':'Create key'}) + + +@login_required +@person_required +def apikey_del(request): + person = request.user.person + choices = [ (k.hash(), str(k)) for k in person.apikeys.all() ] + # + class KeyDeleteForm(forms.Form): + hash = forms.ChoiceField(label='Key', choices=choices) + def clean_key(self): + hash = self.cleaned_data['hash'] + key = PersonalApiKey.validate_key(hash) + if key and key.person == request.user.person: + return hash + else: + raise ValidationError("Bad key value") + # + if request.method == 'POST': + form = KeyDeleteForm(request.POST) + if form.is_valid(): + hash = form.data['hash'] + key = PersonalApiKey.validate_key(hash) + key.delete() + messages.success(request, "Deleted key %s" % hash) + return redirect('ietf.ietfauth.views.apikey_index') + else: + messages.error(request, "Key validation failed; key not deleted") + else: + form = KeyDeleteForm(request.GET) + return render(request, 'form.html', {'form':form, 'title':"Delete a personal API key", 'description':'', 'button':'Delete key'}) diff --git a/ietf/person/admin.py b/ietf/person/admin.py index c50536280..8659ac3f1 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin -from ietf.person.models import Email, Alias, Person, PersonHistory +from ietf.person.models import Email, Alias, Person, PersonHistory, PersonalApiKey from ietf.person.name import name_parts class EmailAdmin(admin.ModelAdmin): @@ -43,4 +43,9 @@ class PersonHistoryAdmin(admin.ModelAdmin): search_fields = ['name', 'ascii'] admin.site.register(PersonHistory, PersonHistoryAdmin) - +class PersonalApiKeyAdmin(admin.ModelAdmin): + list_display = ['id', 'person', 'created', 'endpoint', 'valid', 'count', 'latest', ] + list_filter = ['endpoint', 'created', ] + raw_id_fields = ['person', ] + search_fields = ['person__name', ] +admin.site.register(PersonalApiKey, PersonalApiKeyAdmin) diff --git a/ietf/person/models.py b/ietf/person/models.py index 217720fbe..8e640152f 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -3,6 +3,9 @@ import datetime import email.utils import email.header +import six +import uuid + from hashids import Hashids from urlparse import urljoin @@ -274,3 +277,48 @@ class Email(models.Model): return return self.address + +# "{key.id}{salt}{hash} +KEY_STRUCT = "i12s32s" + +def salt(): + return uuid.uuid4().bytes[:12] + +API_KEY_ENDPOINTS = [ + ("/api/submit", "/api/submit"), + ("/api/iesg/discuss", "/api/iesg/discuss"), +] + +class PersonalApiKey(models.Model): + person = models.ForeignKey(Person, related_name='apikeys') + endpoint = models.CharField(max_length=128, null=False, blank=False, choices=API_KEY_ENDPOINTS) + created = models.DateTimeField(default=datetime.datetime.now, null=False) + valid = models.BooleanField(default=True) + salt = models.BinaryField(default=salt, max_length=12, null=False, blank=False) + count = models.IntegerField(default=0, null=False, blank=False) + latest = models.DateTimeField(blank=True, null=True) + + @classmethod + def validate_key(cls, s): + import struct, hashlib, base64 + key = base64.urlsafe_b64decode(six.binary_type(s)) + id, salt, hash = struct.unpack(KEY_STRUCT, key) + k = cls.objects.get(id=id) + check = hashlib.sha256() + for v in (str(id), str(k.person.id), k.created.isoformat(), k.endpoint, str(k.valid), salt, settings.SECRET_KEY): + check.update(v) + return k if check.digest() == hash else None + + def hash(self): + import struct, hashlib, base64 + if not hasattr(self, '_cached_hash'): + hash = hashlib.sha256() + # Hash over: ( id, person, created, endpoint, valid, salt, secret ) + for v in (str(self.id), str(self.person.id), self.created.isoformat(), self.endpoint, str(self.valid), self.salt, settings.SECRET_KEY): + hash.update(v) + key = struct.pack(KEY_STRUCT, self.id, six.binary_type(self.salt), hash.digest()) + self._cached_hash = base64.urlsafe_b64encode(key) + return self._cached_hash + + def __unicode__(self): + return "%s (%s): %s ..." % (self.endpoint, self.created.strftime("%Y-%m-%d %H:%M"), self.hash()[:16]) diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 11ef747bf..2d23c9069 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,7 +6,7 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import (Person, Email, Alias, PersonHistory) +from ietf.person.models import (Person, Email, Alias, PersonHistory, PersonalApiKey) from ietf.utils.resources import UserResource @@ -82,3 +82,23 @@ class PersonHistoryResource(ModelResource): "user": ALL_WITH_RELATIONS, } api.person.register(PersonHistoryResource()) + + +class PersonalApiKeyResource(ModelResource): + person = ToOneField(PersonResource, 'person') + class Meta: + queryset = PersonalApiKey.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'personalapikey' + filtering = { + "id": ALL, + "endpoint": ALL, + "created": ALL, + "valid": ALL, + "salt": ALL, + "count": ALL, + "latest": ALL, + "person": ALL_WITH_RELATIONS, + } +api.person.register(PersonalApiKeyResource()) diff --git a/ietf/templates/base.html b/ietf/templates/base.html index 0d5f7bc97..1287563b3 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -108,7 +108,7 @@ {% endif %} - {% block content %}{% endblock %} + {% block content %}{{ content|safe }}{% endblock %} {% block content_end %}{% endblock %} {% if request.COOKIES.left_menu != "off" and not hide_menu %} diff --git a/ietf/templates/base/menu_user.html b/ietf/templates/base/menu_user.html index f3f0d07b2..47364c8de 100644 --- a/ietf/templates/base/menu_user.html +++ b/ietf/templates/base/menu_user.html @@ -18,6 +18,7 @@
  • Sign out
  • Account info
  • Preferences
    • +
  • API keys
  • Change password
  • Change username
    • {% else %} diff --git a/ietf/templates/form.html b/ietf/templates/form.html new file mode 100644 index 000000000..a15340209 --- /dev/null +++ b/ietf/templates/form.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin %} +{% load staticfiles %} +{% load bootstrap3 %} + +{% block title %}{{ title|striptags }}{% endblock %} + +{% block pagehead %} + +{% endblock %} + +{% block content %} + {% origin %} +

    {{ title|safe }}

    + +

    + {{ description|safe }} +

    + +
    + {% csrf_token %} + + {% bootstrap_form form %} + + {% buttons %} + + {% endbuttons %} +
    +{% endblock %} + +{% block js %} + +{% endblock %} diff --git a/ietf/templates/ietfauth/apikeys.html b/ietf/templates/ietfauth/apikeys.html new file mode 100644 index 000000000..c6cd30d8d --- /dev/null +++ b/ietf/templates/ietfauth/apikeys.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin %} + +{% load widget_tweaks bootstrap3 %} + +{% load person_filters %} + +{% block title %}API keys for {{ user }}{% endblock %} + +{% block content %} + {% origin %} +

    API keys for {{ user.username }}

    + + {% csrf_token %} +
    + +
    +
    + + {% for key in person.apikeys.all %} + {% if forloop.first %} + + {% endif %} + + + + + + + + {% empty %} + + + {% endfor %} +
    KeyCreatedEndpointValid 
    {{ key.hash }}{{ key.created }} {{ key.endpoint }} {{ key.valid }} + {% if key.valid %} + Delete + {% endif %} +
    You have no personal API keys.
    + Get a new personal API key +
    +
    + +
    + +{% endblock %} diff --git a/ietf/templates/person/profile.html b/ietf/templates/person/profile.html index 5746b350e..a10f61f3d 100644 --- a/ietf/templates/person/profile.html +++ b/ietf/templates/person/profile.html @@ -34,8 +34,8 @@ -
    - {% if person.role_set.exists %} + {% if person.role_set.exists %} +

    Roles

    {% for role in person.role_set.all|active_roles %} @@ -54,8 +54,10 @@ {{ person.first_name }} has no active roles as of {{ today }}. {% endfor %}
    - {% endif %} -
    +
    + {% endif %} + +

    RFCs

    diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 2718d4ec2..666508870 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -15,3 +15,55 @@ def skip_coverage(f, *args, **kwargs): return result else: return f(*args, **kwargs) + +@decorator +def person_required(f, request, *args, **kwargs): + from ietf.person.models import Person + from django.shortcuts import render + if not request.user.is_authenticated: + raise ValueError("The @person_required decorator should be called after @login_required.") + try: + request.user.person + except Person.DoesNotExist: + return render(request, 'registration/missing_person.html') + return f(request, *args, **kwargs) + +@decorator +def verify_user_api_key(f, request, *args, **kwargs): + from ietf.person.models import Person, PersonalApiKey + from django.shortcuts import render + if not request.user.is_authenticated: + raise ValueError("The @verify_user_api_key decorator should be called after @login_required.") + try: + person = request.user.person + except Person.DoesNotExist: + return render(request, 'registration/missing_person.html') + if request.method == 'POST': + hash = request.POST['apikey'] + elif request.method == 'GET': + hash = request.GET['apikey'] + else: + return render(request, 'base.html', { + 'content': """ +

    Missing API key

    + +

    + There is no apikey provided with this call. + Please create a valid Personal API key and use that with your request. +

    + """, + }) + key = PersonalApiKey.validate_key(hash) + if key and key.person == person: + return f(request, key, *args, **kwargs) + else: + return render(request, 'base.html', { + 'content': """ +

    Bad API key

    + +

    + The API key provided with this cal is invalid. + Please create a valid Personal API key and use that with your request. +

    + """, + }) From 383b8b16b96f5e14a2f18cd1ca2ddc7cd6c18408 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sat, 16 Dec 2017 18:27:06 +0000 Subject: [PATCH 2/6] Corrected daily and hourly cron script descriptions. - Legacy-Id: 14425 --- bin/daily | 3 +-- bin/hourly | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/daily b/bin/daily index a00489122..b8b662924 100755 --- a/bin/daily +++ b/bin/daily @@ -3,8 +3,7 @@ # Nightly datatracker jobs. # # This script is expected to be triggered by cron from -# $DTDIR/etc/cron.d/datatracker which should be symlinked from -# /etc/cron.d/ +# /etc/cron.d/datatracker # Run the hourly jobs first $DTDIR/bin/hourly diff --git a/bin/hourly b/bin/hourly index e73a51a8f..e9f496b7c 100755 --- a/bin/hourly +++ b/bin/hourly @@ -3,8 +3,7 @@ # Hourly datatracker jobs # # This script is expected to be triggered by cron from -# $DTDIR/etc/cron.d/datatracker which should be symlinked from -# /etc/cron.d/ +# /etc/cron.d/datatracker DTDIR=/a/www/ietf-datatracker/web cd $DTDIR/ From e7209c6e5004cb1de19fe2d40e444c101c00c8c8 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sat, 16 Dec 2017 18:37:52 +0000 Subject: [PATCH 3/6] Added a new personal event table to keep track of personal API key logins, and a management command to send out reports about activity to users with API keys. Added a weekly cronjob script to trigger weekly reports, and a monthly script for future use. Added a @require_api_key decorator to validate API keys for API key views and log in the API key owner. Modified the API key management urls to use create and disable rather than add and delete. Updated the API key list view. Added an API placeholder view function for ballot position setting, for test purposes. Added tests for the decorator and management command. - Legacy-Id: 14426 --- bin/monthly | 15 +++ bin/weekly | 20 +++ ietf/api/urls.py | 2 + ietf/doc/views_ballot.py | 21 ++- .../commands/send_apikey_usage_emails.py | 51 ++++++++ ietf/ietfauth/tests.py | 122 +++++++++++++++--- ietf/ietfauth/urls.py | 4 +- ietf/ietfauth/views.py | 13 +- ietf/person/admin.py | 14 +- ietf/person/migrations/0021_personalapikey.py | 56 ++++++++ ietf/person/models.py | 33 ++++- ietf/person/resources.py | 40 +++++- ietf/settings.py | 2 +- ietf/templates/ietfauth/apikeys.html | 16 ++- ietf/templates/utils/apikey_usage_report.txt | 14 ++ ietf/utils/decorators.py | 83 ++++++------ 16 files changed, 425 insertions(+), 81 deletions(-) create mode 100755 bin/monthly create mode 100755 bin/weekly create mode 100644 ietf/ietfauth/management/commands/send_apikey_usage_emails.py create mode 100644 ietf/person/migrations/0021_personalapikey.py create mode 100644 ietf/templates/utils/apikey_usage_report.txt diff --git a/bin/monthly b/bin/monthly new file mode 100755 index 000000000..5c15827e1 --- /dev/null +++ b/bin/monthly @@ -0,0 +1,15 @@ +#!/bin/bash + +# Weekly datatracker jobs. +# +# This script is expected to be triggered by cron from +# /etc/cron.d/datatracker + +DTDIR=/a/www/ietf-datatracker/web +cd $DTDIR/ + +# Set up the virtual environment +source $DTDIR/env/bin/activate + +logger -p user.info -t cron "Running $DTDIR/bin/monthly" + diff --git a/bin/weekly b/bin/weekly new file mode 100755 index 000000000..e5ab321fb --- /dev/null +++ b/bin/weekly @@ -0,0 +1,20 @@ +#!/bin/bash + +# Weekly datatracker jobs. +# +# This script is expected to be triggered by cron from +# /etc/cron.d/datatracker + +DTDIR=/a/www/ietf-datatracker/web +cd $DTDIR/ + +# Set up the virtual environment +source $DTDIR/env/bin/activate + +logger -p user.info -t cron "Running $DTDIR/bin/weekly" + + +# Send out weekly summaries of apikey usage + +$DTDIR/ietf/manage.py send_apikey_usage_emails + diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 7cb85e8f4..1c2660137 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -3,6 +3,7 @@ from django.conf.urls import include from ietf import api +from ietf.doc import views_ballot from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url @@ -15,6 +16,7 @@ urlpatterns = [ # Custom API endpoints url(r'^notify/meeting/import_recordings/(?P[a-z0-9-]+)/?$', meeting_views.api_import_recordings), url(r'^submit/?$', submit_views.api_submit), + url(r'^iesg/position', views_ballot.api_set_position), ] # Additional (standard) Tastypie endpoints for n,a in api._api_list: diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 761c3f870..e21221210 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -3,12 +3,14 @@ import datetime, json -from django.http import HttpResponseForbidden, HttpResponseRedirect, Http404 -from django.shortcuts import render, get_object_or_404, redirect -from django.urls import reverse as urlreverse -from django.template.loader import render_to_string from django import forms from django.conf import settings +from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404 +from django.shortcuts import render, get_object_or_404, redirect +from django.template.loader import render_to_string +from django.urls import reverse as urlreverse +from django.views.decorators.csrf import csrf_exempt + import debug # pyflakes:ignore @@ -23,12 +25,13 @@ from ietf.doc.mails import ( email_ballot_deferred, email_ballot_undeferred, from ietf.doc.lastcall import request_last_call from ietf.iesg.models import TelechatDate from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream +from ietf.mailtrigger.utils import gather_address_lists +from ietf.mailtrigger.forms import CcSelectForm from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName from ietf.person.models import Person from ietf.utils.mail import send_mail_text, send_mail_preformatted -from ietf.mailtrigger.utils import gather_address_lists -from ietf.mailtrigger.forms import CcSelectForm +from ietf.utils.decorators import require_user_api_key BALLOT_CHOICES = (("yes", "Yes"), ("noobj", "No Objection"), @@ -233,6 +236,12 @@ def edit_position(request, name, ballot_id): blocking_positions=json.dumps(blocking_positions), )) +@require_user_api_key +@role_required('Area Director', 'Secretariat') +@csrf_exempt +def api_set_position(request): + return HttpResponse("Done", status=200, content_type='text/plain') + @role_required('Area Director','Secretariat') def send_ballot_comment(request, name, ballot_id): diff --git a/ietf/ietfauth/management/commands/send_apikey_usage_emails.py b/ietf/ietfauth/management/commands/send_apikey_usage_emails.py new file mode 100644 index 000000000..2718ef02a --- /dev/null +++ b/ietf/ietfauth/management/commands/send_apikey_usage_emails.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2017, All Rights Reserved +from __future__ import print_function, unicode_literals + +import datetime + +from textwrap import dedent + +from django.conf import settings +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.person.models import PersonalApiKey, PersonApiKeyEvent +from ietf.utils.mail import send_mail + + +class Command(BaseCommand): + """ + Send out emails to all persons who have personal API keys about usage. + + Usage is show over the given period, where the default period is 7 days. + """ + + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument('-d', '--days', dest='days', type=int, default=7, + help='The period over which to show usage.') + + def handle(self, *filenames, **options): + """ + """ + + self.verbosity = int(options.get('verbosity')) + days = options.get('days') + + keys = PersonalApiKey.objects.filter(valid=True) + for key in keys: + earliest = datetime.datetime.now() - datetime.timedelta(days=days) + events = PersonApiKeyEvent.objects.filter(key=key, time__gt=earliest) + count = events.count() + events = events[:32] + if count: + key_name = key.hash()[:8] + subject = "API key usage for key '%s' for the last %s days" %(key_name, days) + to = key.person.email_address() + frm = settings.DEFAULT_FROM_EMAIL + send_mail(None, to, frm, subject, 'utils/apikey_usage_report.txt', {'person':key.person, + 'days':days, 'key':key, 'key_name':key_name, 'count':count, 'events':events, } ) + diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index f261c3075..34d587298 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -17,9 +17,10 @@ from ietf.utils.test_utils import TestCase, login_testing_unauthorized, uniconte from ietf.utils.test_data import make_test_data, make_review_data from ietf.utils.mail import outbox, empty_outbox from ietf.group.models import Group, Role, RoleName +from ietf.group.factories import GroupFactory from ietf.ietfauth.htpasswd import update_htpasswd_file from ietf.mailinglists.models import Subscribed -from ietf.person.models import Person, Email +from ietf.person.models import Person, Email, PersonalApiKey, PERSON_API_KEY_ENDPOINTS from ietf.person.factories import PersonFactory from ietf.review.models import ReviewWish, UnavailablePeriod from ietf.utils.decorators import skip_coverage @@ -497,7 +498,7 @@ class IetfAuthTests(TestCase): self.assertEqual(prev, user) self.assertTrue(user.check_password(u'password')) - def test_apikey(self): + def test_apikey_management(self): person = PersonFactory() url = urlreverse('ietf.ietfauth.views.apikey_index') @@ -511,33 +512,33 @@ class IetfAuthTests(TestCase): self.assertContains(r, 'Get a new personal API key') # Check the add key form content - url = urlreverse('ietf.ietfauth.views.apikey_add') + url = urlreverse('ietf.ietfauth.views.apikey_create') r = self.client.get(url) self.assertContains(r, 'Create a new personal API key') self.assertContains(r, 'Endpoint') # Add 2 keys - r = self.client.post(url, {'endpoint': '/api/submit'}) - self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) - r = self.client.post(url, {'endpoint': '/api/iesg/discuss'}) - self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + for endpoint, display in PERSON_API_KEY_ENDPOINTS: + r = self.client.post(url, {'endpoint': endpoint}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) # Check api key list content url = urlreverse('ietf.ietfauth.views.apikey_index') r = self.client.get(url) - self.assertContains(r, '/api/submit') - self.assertContains(r, '/api/iesg/discuss') + for endpoint, display in PERSON_API_KEY_ENDPOINTS: + self.assertContains(r, endpoint) q = PyQuery(r.content) - self.assertEqual(len(q('td code')), 2) + self.assertEqual(len(q('td code')), len(PERSON_API_KEY_ENDPOINTS)) # hash + self.assertEqual(len(q('td a:contains("Disable")')), len(PERSON_API_KEY_ENDPOINTS)) # Get one of the keys key = person.apikeys.first() - # Check the delete key form content - url = urlreverse('ietf.ietfauth.views.apikey_del') + # Check the disable key form content + url = urlreverse('ietf.ietfauth.views.apikey_disable') r = self.client.get(url) - self.assertContains(r, 'Delete a personal API key') + self.assertContains(r, 'Disable a personal API key') self.assertContains(r, 'Key') # Delete a key @@ -547,7 +548,98 @@ class IetfAuthTests(TestCase): # Check the api key list content again url = urlreverse('ietf.ietfauth.views.apikey_index') r = self.client.get(url) - self.assertNotContains(r, key.endpoint) q = PyQuery(r.content) - self.assertEqual(len(q('td code')), 1) + self.assertEqual(len(q('td code')), len(PERSON_API_KEY_ENDPOINTS)) # key hash + self.assertEqual(len(q('td a:contains("Disable")')), len(PERSON_API_KEY_ENDPOINTS)-1) + + def test_apikey_usage(self): + BAD_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + person = PersonFactory() + area = GroupFactory(type_id='area') + area.role_set.create(name_id='ad', person=person, email=person.email()) + + url = urlreverse('ietf.ietfauth.views.apikey_create') + # Check that the url is protected, then log in + login_testing_unauthorized(self, person.user.username, url) + + # Add keys + for endpoint, display in PERSON_API_KEY_ENDPOINTS: + r = self.client.post(url, {'endpoint': endpoint}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + + for key in person.apikeys.all()[:3]: + url = key.endpoint + + # successful access + r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',}) + self.assertEqual(r.status_code, 200) + + # bad method + r = self.client.put(url, {'apikey':key.hash()}) + self.assertEqual(r.status_code, 405) + + # missing apikey + r = self.client.post(url, {'dummy':'dummy',}) + self.assertEqual(r.status_code, 400) + self.assertIn('Missing apikey parameter', unicontent(r)) + + # invalid apikey + r = self.client.post(url, {'apikey':BAD_KEY, 'dummy':'dummy',}) + self.assertEqual(r.status_code, 400) + self.assertIn('Invalid apikey', unicontent(r)) + + # too long since regular login + person.user.last_login = datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS+1) + person.user.save() + r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',}) + self.assertEqual(r.status_code, 400) + self.assertIn('Too long since last regular login', unicontent(r)) + person.user.last_login = datetime.datetime.now() + person.user.save() + + # endpoint mismatch + key2 = PersonalApiKey.objects.create(person=person, endpoint='/') + r = self.client.post(url, {'apikey':key2.hash(), 'dummy':'dummy',}) + self.assertEqual(r.status_code, 400) + self.assertIn('Apikey endpoint mismatch', unicontent(r)) + key2.delete() + + def test_send_apikey_report(self): + from ietf.ietfauth.management.commands.send_apikey_usage_emails import Command + from ietf.utils.mail import outbox, empty_outbox + + person = PersonFactory() + + url = urlreverse('ietf.ietfauth.views.apikey_create') + # Check that the url is protected, then log in + login_testing_unauthorized(self, person.user.username, url) + + # Add keys + for endpoint, display in PERSON_API_KEY_ENDPOINTS: + r = self.client.post(url, {'endpoint': endpoint}) + self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) + + # Use the endpoints (the form content will not be acceptable, but the + # apikey usage will be registered) + count = 2 + # avoid usage across dates + if datetime.datetime.now().time() > datetime.time(hour=23, minute=59, second=58): + time.sleep(2) + for i in range(count): + for key in person.apikeys.all(): + url = key.endpoint + self.client.post(url, {'apikey':key.hash(), 'dummy': 'dummy', }) + date = str(datetime.date.today()) + + empty_outbox() + cmd = Command() + cmd.handle(verbosity=0, days=7) + + self.assertEqual(len(outbox), len(PERSON_API_KEY_ENDPOINTS)) + for mail in outbox: + body = mail.get_payload() + self.assertIn("API key usage", mail['subject']) + self.assertIn(" %s times" % count, body) + self.assertIn(date, body) diff --git a/ietf/ietfauth/urls.py b/ietf/ietfauth/urls.py index b586d284b..fb1f84db5 100644 --- a/ietf/ietfauth/urls.py +++ b/ietf/ietfauth/urls.py @@ -8,8 +8,8 @@ from ietf.utils.urls import url urlpatterns = [ url(r'^$', views.index), url(r'^apikey/?$', views.apikey_index), - url(r'^apikey/add/?$', views.apikey_add), - url(r'^apikey/del/?$', views.apikey_del), + url(r'^apikey/add/?$', views.apikey_create), + url(r'^apikey/del/?$', views.apikey_disable), url(r'^confirmnewemail/(?P[^/]+)/$', views.confirm_new_email), url(r'^create/$', views.create_account), url(r'^create/confirm/(?P[^/]+)/$', views.confirm_account), diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 99f4249bd..e3f549279 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -602,7 +602,7 @@ def apikey_index(request): @login_required @person_required -def apikey_add(request): +def apikey_create(request): class ApiKeyForm(forms.ModelForm): class Meta: model = PersonalApiKey @@ -623,7 +623,7 @@ def apikey_add(request): @login_required @person_required -def apikey_del(request): +def apikey_disable(request): person = request.user.person choices = [ (k.hash(), str(k)) for k in person.apikeys.all() ] # @@ -642,11 +642,12 @@ def apikey_del(request): if form.is_valid(): hash = form.data['hash'] key = PersonalApiKey.validate_key(hash) - key.delete() - messages.success(request, "Deleted key %s" % hash) + key.valid = False + key.save() + messages.success(request, "Disabled key %s" % hash) return redirect('ietf.ietfauth.views.apikey_index') else: - messages.error(request, "Key validation failed; key not deleted") + messages.error(request, "Key validation failed; key not disabled") else: form = KeyDeleteForm(request.GET) - return render(request, 'form.html', {'form':form, 'title':"Delete a personal API key", 'description':'', 'button':'Delete key'}) + return render(request, 'form.html', {'form':form, 'title':"Disable a personal API key", 'description':'', 'button':'Disable key'}) diff --git a/ietf/person/admin.py b/ietf/person/admin.py index 8659ac3f1..86c0c851f 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin -from ietf.person.models import Email, Alias, Person, PersonHistory, PersonalApiKey +from ietf.person.models import Email, Alias, Person, PersonHistory, PersonalApiKey, PersonEvent, PersonApiKeyEvent from ietf.person.name import name_parts class EmailAdmin(admin.ModelAdmin): @@ -49,3 +49,15 @@ class PersonalApiKeyAdmin(admin.ModelAdmin): raw_id_fields = ['person', ] search_fields = ['person__name', ] admin.site.register(PersonalApiKey, PersonalApiKeyAdmin) + +class PersonEventAdmin(admin.ModelAdmin): + list_display = ["id", "person", "time", "type", ] + search_fields = ["person__name", ] + raw_id_fields = ['person', ] +admin.site.register(PersonEvent, PersonEventAdmin) + +class PersonApiKeyEventAdmin(admin.ModelAdmin): + list_display = ["id", "person", "time", "type", "key"] + search_fields = ["person__name", ] + raw_id_fields = ['person', ] +admin.site.register(PersonApiKeyEvent, PersonApiKeyEventAdmin) diff --git a/ietf/person/migrations/0021_personalapikey.py b/ietf/person/migrations/0021_personalapikey.py new file mode 100644 index 000000000..e3e9e4bc3 --- /dev/null +++ b/ietf/person/migrations/0021_personalapikey.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.8 on 2017-12-15 08:19 +from __future__ import unicode_literals + +import datetime +from django.db import migrations, models +import django.db.models.deletion +import ietf.person.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0020_auto_20170701_0325'), + ] + + operations = [ + migrations.CreateModel( + name='PersonalApiKey', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('endpoint', models.CharField(choices=[(b'/api/submit', b'/api/submit'), (b'/api/iesg/position', b'/api/iesg/position')], max_length=128)), + ('created', models.DateTimeField(default=datetime.datetime.now)), + ('valid', models.BooleanField(default=True)), + ('salt', models.BinaryField(default=ietf.person.models.salt, max_length=12)), + ('count', models.IntegerField(default=0)), + ('latest', models.DateTimeField(blank=True, null=True)), + ('person', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='apikeys', to='person.Person')), + ], + ), + migrations.CreateModel( + name='PersonEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time', models.DateTimeField(default=datetime.datetime.now, help_text=b'When the event happened')), + ('type', models.CharField(choices=[(b'apikey_login', b'API key login')], max_length=50)), + ('desc', models.TextField()), + ], + options={ + 'ordering': ['-time', '-id'], + }, + ), + migrations.CreateModel( + name='PersonApiKeyEvent', + fields=[ + ('personevent_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='person.PersonEvent')), + ('key', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.PersonalApiKey')), + ], + bases=('person.personevent',), + ), + migrations.AddField( + model_name='personevent', + name='person', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='person.Person'), + ), + ] diff --git a/ietf/person/models.py b/ietf/person/models.py index 8e640152f..16c1ee9c8 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -284,14 +284,14 @@ KEY_STRUCT = "i12s32s" def salt(): return uuid.uuid4().bytes[:12] -API_KEY_ENDPOINTS = [ - ("/api/submit", "/api/submit"), - ("/api/iesg/discuss", "/api/iesg/discuss"), +# Manual maintenance: List all endpoints that use @require_user_api_key here +PERSON_API_KEY_ENDPOINTS = [ + ("/api/iesg/position", "/api/iesg/position"), ] class PersonalApiKey(models.Model): person = models.ForeignKey(Person, related_name='apikeys') - endpoint = models.CharField(max_length=128, null=False, blank=False, choices=API_KEY_ENDPOINTS) + endpoint = models.CharField(max_length=128, null=False, blank=False, choices=PERSON_API_KEY_ENDPOINTS) created = models.DateTimeField(default=datetime.datetime.now, null=False) valid = models.BooleanField(default=True) salt = models.BinaryField(default=salt, max_length=12, null=False, blank=False) @@ -303,7 +303,10 @@ class PersonalApiKey(models.Model): import struct, hashlib, base64 key = base64.urlsafe_b64decode(six.binary_type(s)) id, salt, hash = struct.unpack(KEY_STRUCT, key) - k = cls.objects.get(id=id) + k = cls.objects.filter(id=id) + if not k.exists(): + return None + k = k.first() check = hashlib.sha256() for v in (str(id), str(k.person.id), k.created.isoformat(), k.endpoint, str(k.valid), salt, settings.SECRET_KEY): check.update(v) @@ -322,3 +325,23 @@ class PersonalApiKey(models.Model): def __unicode__(self): return "%s (%s): %s ..." % (self.endpoint, self.created.strftime("%Y-%m-%d %H:%M"), self.hash()[:16]) + +PERSON_EVENT_CHOICES = [ + ("apikey_login", "API key login"), + ] + +class PersonEvent(models.Model): + person = models.ForeignKey(Person) + time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened") + type = models.CharField(max_length=50, choices=PERSON_EVENT_CHOICES) + desc = models.TextField() + + def __unicode__(self): + return u"%s %s at %s" % (self.person.plain_name(), self.get_type_display().lower(), self.time) + + class Meta: + ordering = ['-time', '-id'] + +class PersonApiKeyEvent(PersonEvent): + key = models.ForeignKey(PersonalApiKey) + diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 2d23c9069..4cd0ce24a 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -6,7 +6,7 @@ from tastypie.cache import SimpleCache from ietf import api -from ietf.person.models import (Person, Email, Alias, PersonHistory, PersonalApiKey) +from ietf.person.models import (Person, Email, Alias, PersonHistory, PersonalApiKey, PersonEvent, PersonApiKeyEvent) from ietf.utils.resources import UserResource @@ -102,3 +102,41 @@ class PersonalApiKeyResource(ModelResource): "person": ALL_WITH_RELATIONS, } api.person.register(PersonalApiKeyResource()) + + +class PersonEventResource(ModelResource): + person = ToOneField(PersonResource, 'person') + class Meta: + queryset = PersonEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'personevent' + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "desc": ALL, + "person": ALL_WITH_RELATIONS, + } +api.person.register(PersonEventResource()) + + +class PersonApiKeyEventResource(ModelResource): + person = ToOneField(PersonResource, 'person') + personevent_ptr = ToOneField(PersonEventResource, 'personevent_ptr') + key = ToOneField(PersonalApiKeyResource, 'key') + class Meta: + queryset = PersonApiKeyEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'personapikeyevent' + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "desc": ALL, + "person": ALL_WITH_RELATIONS, + "personevent_ptr": ALL_WITH_RELATIONS, + "key": ALL_WITH_RELATIONS, + } +api.person.register(PersonApiKeyEventResource()) diff --git a/ietf/settings.py b/ietf/settings.py index 80a583b6a..5cb031c77 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -932,7 +932,7 @@ SILENCED_SYSTEM_CHECKS = [ STATS_NAMES_LIMIT = 25 UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' - +UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS = 30 # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. diff --git a/ietf/templates/ietfauth/apikeys.html b/ietf/templates/ietfauth/apikeys.html index c6cd30d8d..e4fe20ed4 100644 --- a/ietf/templates/ietfauth/apikeys.html +++ b/ietf/templates/ietfauth/apikeys.html @@ -20,16 +20,20 @@ {% for key in person.apikeys.all %} {% if forloop.first %} - + {% endif %} - - + + + - + + + @@ -38,7 +42,7 @@ {% endfor %}
    KeyCreatedEndpointValid 
    EndpointCreatedLatest useCountValid
    {{ key.hash }}{{ key.created }} {{ key.endpoint }} {{ key.created }} {{ key.latest }} {{ key.count }} {{ key.valid }} +
    {{ key.hash }} {% if key.valid %} - Delete + Disable {% endif %}
    - Get a new personal API key + Get a new personal API key
    diff --git a/ietf/templates/utils/apikey_usage_report.txt b/ietf/templates/utils/apikey_usage_report.txt new file mode 100644 index 000000000..a3b7bfd7b --- /dev/null +++ b/ietf/templates/utils/apikey_usage_report.txt @@ -0,0 +1,14 @@ +{% load ietf_filters %}{% filter wordwrap:78 %} +Dear {{ person.plain_name }}, + +This is a summary of API key usage during the last {{ days }} days for the key '{{ key_name }}', +created {{ key.created }} for endpoint {{ key.endpoint }}. + +This API key was used {{ count }} times during this period. Showing {{ events|length }} access times: +{% for e in events %} + {{ e.time }}{% endfor %} + + +Best regards, +The IETF Secretariat +{% endfilter %} diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 666508870..7c8ac3a1e 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -1,10 +1,18 @@ # Copyright The IETF Trust 2016, All Rights Reserved +import datetime + from decorator import decorator from django.conf import settings +from django.contrib.auth import login +from django.http import HttpResponse +from django.shortcuts import render -from test_runner import set_coverage_checking +import debug # pyflakes:ignore + +from ietf.utils.test_runner import set_coverage_checking +from ietf.person.models import Person, PersonalApiKey, PersonApiKeyEvent @decorator def skip_coverage(f, *args, **kwargs): @@ -18,8 +26,6 @@ def skip_coverage(f, *args, **kwargs): @decorator def person_required(f, request, *args, **kwargs): - from ietf.person.models import Person - from django.shortcuts import render if not request.user.is_authenticated: raise ValueError("The @person_required decorator should be called after @login_required.") try: @@ -27,43 +33,44 @@ def person_required(f, request, *args, **kwargs): except Person.DoesNotExist: return render(request, 'registration/missing_person.html') return f(request, *args, **kwargs) - + @decorator -def verify_user_api_key(f, request, *args, **kwargs): - from ietf.person.models import Person, PersonalApiKey - from django.shortcuts import render - if not request.user.is_authenticated: - raise ValueError("The @verify_user_api_key decorator should be called after @login_required.") - try: - person = request.user.person - except Person.DoesNotExist: - return render(request, 'registration/missing_person.html') +def require_user_api_key(f, request, *args, **kwargs): + + def err(code, text): + return HttpResponse(text, status=code, content_type='text/plain') + # Check method and get hash if request.method == 'POST': - hash = request.POST['apikey'] + hash = request.POST.get('apikey') elif request.method == 'GET': - hash = request.GET['apikey'] + hash = request.GET.get('apikey') else: - return render(request, 'base.html', { - 'content': """ -

    Missing API key

    - -

    - There is no apikey provided with this call. - Please create a valid Personal API key and use that with your request. -

    - """, - }) + return err(405, "Method not allowed") + if not hash: + return err(400, "Missing apikey parameter") + # Check hash key = PersonalApiKey.validate_key(hash) - if key and key.person == person: - return f(request, key, *args, **kwargs) - else: - return render(request, 'base.html', { - 'content': """ -

    Bad API key

    - -

    - The API key provided with this cal is invalid. - Please create a valid Personal API key and use that with your request. -

    - """, - }) + if not key: + return err(400, "Invalid apikey") + # Check endpoint + urlpath = request.META.get('PATH_INFO') + if not (urlpath and urlpath == key.endpoint): + return err(400, "Apikey endpoint mismatch") + # Check time since regular login + person = key.person + last_login = person.user.last_login + time_limit = (datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS)) + if last_login < time_limit: + return err(400, "Too long since last regular login") + # Log in + login(request, person.user) + # restore the user.last_login field, so it reflects only gui logins + person.user.last_login = last_login + person.user.save() + # Update stats + key.count += 1 + key.latest = datetime.datetime.now() + key.save() + PersonApiKeyEvent.objects.create(person=person, type='apikey_login', key=key, desc="Logged in with key ID %s, endpoint %s" % (key.id, key.endpoint)) + # Execute decorated function + return f(request, *args, **kwargs) From ec4e2381852f653d0680113f24ed8c6e530fc9f3 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sat, 16 Dec 2017 18:39:26 +0000 Subject: [PATCH 4/6] Added some missing files. - Legacy-Id: 14427 --- ietf/ietfauth/management/__init__.py | 0 ietf/ietfauth/management/commands/__init__.py | 0 .../commands/send_apikey_usage_emails.py | 51 +++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 ietf/ietfauth/management/__init__.py create mode 100644 ietf/ietfauth/management/commands/__init__.py create mode 100644 ietf/utils/management/commands/send_apikey_usage_emails.py diff --git a/ietf/ietfauth/management/__init__.py b/ietf/ietfauth/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/ietfauth/management/commands/__init__.py b/ietf/ietfauth/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ietf/utils/management/commands/send_apikey_usage_emails.py b/ietf/utils/management/commands/send_apikey_usage_emails.py new file mode 100644 index 000000000..2718ef02a --- /dev/null +++ b/ietf/utils/management/commands/send_apikey_usage_emails.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2017, All Rights Reserved +from __future__ import print_function, unicode_literals + +import datetime + +from textwrap import dedent + +from django.conf import settings +from django.core.management.base import BaseCommand + +import debug # pyflakes:ignore + +from ietf.person.models import PersonalApiKey, PersonApiKeyEvent +from ietf.utils.mail import send_mail + + +class Command(BaseCommand): + """ + Send out emails to all persons who have personal API keys about usage. + + Usage is show over the given period, where the default period is 7 days. + """ + + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + parser.add_argument('-d', '--days', dest='days', type=int, default=7, + help='The period over which to show usage.') + + def handle(self, *filenames, **options): + """ + """ + + self.verbosity = int(options.get('verbosity')) + days = options.get('days') + + keys = PersonalApiKey.objects.filter(valid=True) + for key in keys: + earliest = datetime.datetime.now() - datetime.timedelta(days=days) + events = PersonApiKeyEvent.objects.filter(key=key, time__gt=earliest) + count = events.count() + events = events[:32] + if count: + key_name = key.hash()[:8] + subject = "API key usage for key '%s' for the last %s days" %(key_name, days) + to = key.person.email_address() + frm = settings.DEFAULT_FROM_EMAIL + send_mail(None, to, frm, subject, 'utils/apikey_usage_report.txt', {'person':key.person, + 'days':days, 'key':key, 'key_name':key_name, 'count':count, 'events':events, } ) + From b0863c8963b8631a2b08e1b05d00957fc2aa64b7 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sat, 16 Dec 2017 18:43:08 +0000 Subject: [PATCH 5/6] Modified some names. - Legacy-Id: 14428 --- ietf/doc/views_ballot.py | 9 +++++++-- ietf/person/models.py | 2 +- ietf/utils/decorators.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index e21221210..1c4091233 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -31,7 +31,7 @@ from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName from ietf.person.models import Person from ietf.utils.mail import send_mail_text, send_mail_preformatted -from ietf.utils.decorators import require_user_api_key +from ietf.utils.decorators import require_api_key BALLOT_CHOICES = (("yes", "Yes"), ("noobj", "No Objection"), @@ -236,10 +236,15 @@ def edit_position(request, name, ballot_id): blocking_positions=json.dumps(blocking_positions), )) -@require_user_api_key +@require_api_key @role_required('Area Director', 'Secretariat') @csrf_exempt def api_set_position(request): + if request.method == 'POST': + pass + else: + return HttpResponse("Method not allowed", status=405, content_type='text/plain') + return HttpResponse("Done", status=200, content_type='text/plain') diff --git a/ietf/person/models.py b/ietf/person/models.py index 16c1ee9c8..64cf5884d 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -284,7 +284,7 @@ KEY_STRUCT = "i12s32s" def salt(): return uuid.uuid4().bytes[:12] -# Manual maintenance: List all endpoints that use @require_user_api_key here +# Manual maintenance: List all endpoints that use @require_api_key here PERSON_API_KEY_ENDPOINTS = [ ("/api/iesg/position", "/api/iesg/position"), ] diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 7c8ac3a1e..fe9d22e52 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -35,7 +35,7 @@ def person_required(f, request, *args, **kwargs): return f(request, *args, **kwargs) @decorator -def require_user_api_key(f, request, *args, **kwargs): +def require_api_key(f, request, *args, **kwargs): def err(code, text): return HttpResponse(text, status=code, content_type='text/plain') From a08c8dc76f526801dd5264abcd68bf7465b87fe9 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 17 Dec 2017 21:55:27 +0000 Subject: [PATCH 6/6] Added an API endpoint to support automation of IESG ballot position posting, at /api/iesg/position. Added tests for the API endpoint, and updated the apikey validation decorator tests. Tweaked the decorator to handle a weakness found during testing. - Legacy-Id: 14429 --- ietf/doc/tests_ballot.py | 61 +++++++++++++- ietf/doc/views_ballot.py | 166 +++++++++++++++++++++++---------------- ietf/ietfauth/tests.py | 13 ++- ietf/utils/decorators.py | 2 +- 4 files changed, 166 insertions(+), 76 deletions(-) diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 3007c666c..0a3f692e2 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -12,7 +12,7 @@ from ietf.doc.factories import DocumentFactory from ietf.group.models import Group, Role from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate -from ietf.person.models import Person +from ietf.person.models import Person, PersonalApiKey from ietf.utils.test_utils import TestCase, unicontent from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_data import make_test_data @@ -82,6 +82,65 @@ class EditPositionTests(TestCase): self.assertEqual(draft.docevent_set.count(), events_before + 2) self.assertTrue("Ballot comment text updated" in pos.desc) + def test_api_set_position(self): + draft = make_test_data() + url = urlreverse('ietf.doc.views_ballot.api_set_position') + ad = Person.objects.get(name="Areað Irector") + ad.user.last_login = datetime.datetime.now() + ad.user.save() + apikey = PersonalApiKey.objects.create(endpoint=url, person=ad) + + # vote + events_before = draft.docevent_set.count() + + r = self.client.post(url, dict( + apikey=apikey.hash(), + doc=draft.name, + position="discuss", + discuss=" This is a discussion test. \n ", + comment=" This is a test. \n ") + ) + self.assertEqual(r.status_code, 200) + + pos = draft.latest_event(BallotPositionDocEvent, ad=ad) + self.assertEqual(pos.pos.slug, "discuss") + self.assertTrue(" This is a discussion test." in pos.discuss) + self.assertTrue(pos.discuss_time != None) + self.assertTrue(" This is a test." in pos.comment) + self.assertTrue(pos.comment_time != None) + self.assertTrue("New position" in pos.desc) + self.assertEqual(draft.docevent_set.count(), events_before + 3) + + # recast vote + events_before = draft.docevent_set.count() + r = self.client.post(url, dict(apikey=apikey.hash(), doc=draft.name, position="noobj")) + self.assertEqual(r.status_code, 200) + + pos = draft.latest_event(BallotPositionDocEvent, ad=ad) + self.assertEqual(pos.pos.slug, "noobj") + self.assertEqual(draft.docevent_set.count(), events_before + 1) + self.assertTrue("Position for" in pos.desc) + + # clear vote + events_before = draft.docevent_set.count() + r = self.client.post(url, dict(apikey=apikey.hash(), doc=draft.name, position="norecord")) + self.assertEqual(r.status_code, 200) + + pos = draft.latest_event(BallotPositionDocEvent, ad=ad) + self.assertEqual(pos.pos.slug, "norecord") + self.assertEqual(draft.docevent_set.count(), events_before + 1) + self.assertTrue("Position for" in pos.desc) + + # change comment + events_before = draft.docevent_set.count() + r = self.client.post(url, dict(apikey=apikey.hash(), doc=draft.name, position="norecord", comment="New comment.")) + self.assertEqual(r.status_code, 200) + + pos = draft.latest_event(BallotPositionDocEvent, ad=ad) + self.assertEqual(pos.pos.slug, "norecord") + self.assertEqual(draft.docevent_set.count(), events_before + 2) + self.assertTrue("Ballot comment text updated" in pos.desc) + def test_edit_position_as_secretary(self): draft = make_test_data() url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 1c4091233..740754bd3 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -7,6 +7,7 @@ from django import forms from django.conf import settings from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404 from django.shortcuts import render, get_object_or_404, redirect +from django.template.defaultfilters import striptags from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.views.decorators.csrf import csrf_exempt @@ -108,6 +109,74 @@ class EditPositionForm(forms.Form): raise forms.ValidationError("You must enter a non-empty discuss") return entered_discuss +def save_position(form, doc, ballot, ad, login=None): + # save the vote + if login is None: + login = ad + clean = form.cleaned_data + + old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", ad=ad, ballot=ballot) + pos = BallotPositionDocEvent(doc=doc, rev=doc.rev, by=login) + pos.type = "changed_ballot_position" + pos.ballot = ballot + pos.ad = ad + pos.pos = clean["position"] + pos.comment = clean["comment"].rstrip() + pos.comment_time = old_pos.comment_time if old_pos else None + pos.discuss = clean["discuss"].rstrip() + if not pos.pos.blocking: + pos.discuss = "" + pos.discuss_time = old_pos.discuss_time if old_pos else None + + changes = [] + added_events = [] + # possibly add discuss/comment comments to history trail + # so it's easy to see what's happened + old_comment = old_pos.comment if old_pos else "" + if pos.comment != old_comment: + pos.comment_time = pos.time + changes.append("comment") + + if pos.comment: + e = DocEvent(doc=doc, rev=doc.rev) + e.by = ad # otherwise we can't see who's saying it + e.type = "added_comment" + e.desc = "[Ballot comment]\n" + pos.comment + added_events.append(e) + + old_discuss = old_pos.discuss if old_pos else "" + if pos.discuss != old_discuss: + pos.discuss_time = pos.time + changes.append("discuss") + + if pos.pos.blocking: + e = DocEvent(doc=doc, rev=doc.rev, by=login) + e.by = ad # otherwise we can't see who's saying it + e.type = "added_comment" + e.desc = "[Ballot %s]\n" % pos.pos.name.lower() + e.desc += pos.discuss + added_events.append(e) + + # figure out a description + if not old_pos and pos.pos.slug != "norecord": + pos.desc = u"[Ballot Position Update] New position, %s, has been recorded for %s" % (pos.pos.name, pos.ad.plain_name()) + elif old_pos and pos.pos != old_pos.pos: + pos.desc = "[Ballot Position Update] Position for %s has been changed to %s from %s" % (pos.ad.plain_name(), pos.pos.name, old_pos.pos.name) + + if not pos.desc and changes: + pos.desc = u"Ballot %s text updated for %s" % (u" and ".join(changes), ad.plain_name()) + + # only add new event if we actually got a change + if pos.desc: + if login != ad: + pos.desc += u" by %s" % login.plain_name() + + pos.save() + + for e in added_events: + e.save() # save them after the position is saved to get later id for sorting order + + @role_required('Area Director','Secretariat') def edit_position(request, name, ballot_id): """Vote and edit discuss and comment on document as Area Director.""" @@ -128,8 +197,6 @@ def edit_position(request, name, ballot_id): raise Http404 ad = get_object_or_404(Person, pk=ad_id) - old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", ad=ad, ballot=ballot) - if request.method == 'POST': if not has_role(request.user, "Secretariat") and not ad.role_set.filter(name="ad", group__type="area", group__state="active"): # prevent pre-ADs from voting @@ -137,68 +204,7 @@ def edit_position(request, name, ballot_id): form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type) if form.is_valid(): - # save the vote - clean = form.cleaned_data - - pos = BallotPositionDocEvent(doc=doc, rev=doc.rev, by=login) - pos.type = "changed_ballot_position" - pos.ballot = ballot - pos.ad = ad - pos.pos = clean["position"] - pos.comment = clean["comment"].rstrip() - pos.comment_time = old_pos.comment_time if old_pos else None - pos.discuss = clean["discuss"].rstrip() - if not pos.pos.blocking: - pos.discuss = "" - pos.discuss_time = old_pos.discuss_time if old_pos else None - - changes = [] - added_events = [] - # possibly add discuss/comment comments to history trail - # so it's easy to see what's happened - old_comment = old_pos.comment if old_pos else "" - if pos.comment != old_comment: - pos.comment_time = pos.time - changes.append("comment") - - if pos.comment: - e = DocEvent(doc=doc, rev=doc.rev) - e.by = ad # otherwise we can't see who's saying it - e.type = "added_comment" - e.desc = "[Ballot comment]\n" + pos.comment - added_events.append(e) - - old_discuss = old_pos.discuss if old_pos else "" - if pos.discuss != old_discuss: - pos.discuss_time = pos.time - changes.append("discuss") - - if pos.pos.blocking: - e = DocEvent(doc=doc, rev=doc.rev, by=login) - e.by = ad # otherwise we can't see who's saying it - e.type = "added_comment" - e.desc = "[Ballot %s]\n" % pos.pos.name.lower() - e.desc += pos.discuss - added_events.append(e) - - # figure out a description - if not old_pos and pos.pos.slug != "norecord": - pos.desc = u"[Ballot Position Update] New position, %s, has been recorded for %s" % (pos.pos.name, pos.ad.plain_name()) - elif old_pos and pos.pos != old_pos.pos: - pos.desc = "[Ballot Position Update] Position for %s has been changed to %s from %s" % (pos.ad.plain_name(), pos.pos.name, old_pos.pos.name) - - if not pos.desc and changes: - pos.desc = u"Ballot %s text updated for %s" % (u" and ".join(changes), ad.plain_name()) - - # only add new event if we actually got a change - if pos.desc: - if login != ad: - pos.desc += u" by %s" % login.plain_name() - - pos.save() - - for e in added_events: - e.save() # save them after the position is saved to get later id for sorting order + save_position(form, doc, ballot, ad, login) if request.POST.get("send_mail"): qstr="" @@ -213,6 +219,7 @@ def edit_position(request, name, ballot_id): return HttpResponseRedirect(return_to_url) else: initial = {} + old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", ad=ad, ballot=ballot) if old_pos: initial['position'] = old_pos.pos.slug initial['discuss'] = old_pos.discuss @@ -240,10 +247,37 @@ def edit_position(request, name, ballot_id): @role_required('Area Director', 'Secretariat') @csrf_exempt def api_set_position(request): + def err(code, text): + return HttpResponse(text, status=code, content_type='text/plain') if request.method == 'POST': - pass + ad = request.user.person + name = request.POST.get('doc') + if not name: + return err(400, "Missing document name") + try: + doc = Document.objects.get(docalias__name=name) + except Document.DoesNotExist: + return err(404, "Document not found") + position_names = BallotPositionName.objects.values_list('slug', flat=True) + position = request.POST.get('position') + if not position: + return err(400, "Missing parameter: position, one of: %s " % ','.join(position_names)) + if not position in position_names: + return err(400, "Bad position name, must be one of: %s " % ','.join(position_names)) + ballot = doc.active_ballot() + if not ballot: + return err(404, "No open ballot found") + form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type) + if form.is_valid(): + save_position(form, doc, ballot, ad) + else: + debug.type('form.errors') + debug.show('form.errors') + errors = form.errors + summary = ','.join([ "%s: %s" % (f, striptags(errors[f])) for f in errors ]) + return err(400, "Form not valid: %s" % summary) else: - return HttpResponse("Method not allowed", status=405, content_type='text/plain') + return err(405, "Method not allowed") return HttpResponse("Done", status=200, content_type='text/plain') diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index 34d587298..fba73e529 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -1,5 +1,6 @@ +# Copyright The IETF Trust 2017, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals +from __future__ import unicode_literals, print_function import os, shutil, time, datetime from urlparse import urlsplit @@ -101,7 +102,7 @@ class IetfAuthTests(TestCase): if l.startswith(username + ":"): return True with open(settings.HTPASSWD_FILE) as f: - print f.read() + print(f.read()) return False @@ -552,7 +553,7 @@ class IetfAuthTests(TestCase): self.assertEqual(len(q('td code')), len(PERSON_API_KEY_ENDPOINTS)) # key hash self.assertEqual(len(q('td a:contains("Disable")')), len(PERSON_API_KEY_ENDPOINTS)-1) - def test_apikey_usage(self): + def test_apikey_errors(self): BAD_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" person = PersonFactory() @@ -571,10 +572,6 @@ class IetfAuthTests(TestCase): for key in person.apikeys.all()[:3]: url = key.endpoint - # successful access - r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',}) - self.assertEqual(r.status_code, 200) - # bad method r = self.client.put(url, {'apikey':key.hash()}) self.assertEqual(r.status_code, 405) @@ -638,7 +635,7 @@ class IetfAuthTests(TestCase): self.assertEqual(len(outbox), len(PERSON_API_KEY_ENDPOINTS)) for mail in outbox: - body = mail.get_payload() + body = mail.get_payload(decode=True).decode('utf-8') self.assertIn("API key usage", mail['subject']) self.assertIn(" %s times" % count, body) self.assertIn(date, body) diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index fe9d22e52..ef7428664 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -60,7 +60,7 @@ def require_api_key(f, request, *args, **kwargs): person = key.person last_login = person.user.last_login time_limit = (datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS)) - if last_login < time_limit: + if last_login == None or last_login < time_limit: return err(400, "Too long since last regular login") # Log in login(request, person.user)