diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 7506b6480..ba8ad4c91 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -5,7 +5,6 @@ from urllib import urlencode from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -13,76 +12,12 @@ import tastypie import tastypie.resources from tastypie.api import Api from tastypie.bundle import Bundle -from tastypie.serializers import Serializer as BaseSerializer from tastypie.exceptions import ApiFieldError +from tastypie.serializers import Serializer # pyflakes:ignore (we're re-exporting this) from tastypie.fields import ApiField _api_list = [] -class ModelResource(tastypie.resources.ModelResource): - def generate_cache_key(self, *args, **kwargs): - """ - Creates a unique-enough cache key. - - This is based off the current api_name/resource_name/args/kwargs. - """ - #smooshed = ["%s=%s" % (key, value) for key, value in kwargs.items()] - smooshed = urlencode(kwargs) - - # Use a list plus a ``.join()`` because it's faster than concatenation. - return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed) - -class Serializer(BaseSerializer): - def to_html(self, data, options=None): - """ - Reserved for future usage. - - The desire is to provide HTML output of a resource, making an API - available to a browser. This is on the TODO list but not currently - implemented. - """ - from django.template.loader import render_to_string - - options = options or {} - - serialized = self.to_simple_html(data, options) - return render_to_string("api/base.html", {"data": serialized}) - - def to_simple_html(self, data, options): - """ - """ - from django.template.loader import render_to_string - # - if isinstance(data, (list, tuple)): - return render_to_string("api/listitem.html", {"data": [self.to_simple_html(item, options) for item in data]}) - if isinstance(data, dict): - return render_to_string("api/dictitem.html", {"data": dict((key, self.to_simple_html(val, options)) for (key, val) in data.items())}) - elif isinstance(data, Bundle): - return render_to_string("api/dictitem.html", {"data":dict((key, self.to_simple_html(val, options)) for (key, val) in data.data.items())}) - elif hasattr(data, 'dehydrated_type'): - if getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == False: - return render_to_string("api/relitem.html", {"fk": data.fk_resource, "val": self.to_simple_html(data.value, options)}) - elif getattr(data, 'dehydrated_type', None) == 'related' and data.is_m2m == True: - render_to_string("api/listitem.html", {"data": [self.to_simple_html(bundle, options) for bundle in data.m2m_bundles]}) - else: - return self.to_simple_html(data.value, options) - elif isinstance(data, datetime.datetime): - return self.format_datetime(data) - elif isinstance(data, datetime.date): - return self.format_date(data) - elif isinstance(data, datetime.time): - return self.format_time(data) - elif isinstance(data, bool): - return data - elif isinstance(data, (six.integer_types, float)): - return data - elif data is None: - return None - elif isinstance(data, basestring) and data.startswith("/api/v1/"): # XXX Will not work for Python 3 - return render_to_string("api/relitem.html", {"fk": data, "val": data.split('/')[-2]}) - else: - return force_text(data) - for _app in settings.INSTALLED_APPS: _module_dict = globals() if '.' in _app: @@ -116,6 +51,20 @@ def autodiscover(): if module_has_submodule(mod, "resources"): raise +class ModelResource(tastypie.resources.ModelResource): + def generate_cache_key(self, *args, **kwargs): + """ + Creates a unique-enough cache key. + + This is based off the current api_name/resource_name/args/kwargs. + """ + #smooshed = ["%s=%s" % (key, value) for key, value in kwargs.items()] + smooshed = urlencode(kwargs) + + # Use a list plus a ``.join()`` because it's faster than concatenation. + return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed) + + TIMEDELTA_REGEX = re.compile('^(?P\d+d)?\s?(?P\d+h)?\s?(?P\d+m)?\s?(?P\d+s?)$') class TimedeltaField(ApiField): diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py new file mode 100644 index 000000000..11a61f1d3 --- /dev/null +++ b/ietf/api/serializer.py @@ -0,0 +1,258 @@ +import hashlib +import json + +from django.core.cache import cache +from django.core.exceptions import ObjectDoesNotExist, FieldError +from django.core.serializers.json import Serializer +from django.http import HttpResponse +from django.utils.encoding import smart_text +from django.db.models import Field +from django.db.models.query import QuerySet +from django.db.models.signals import post_save, post_delete, m2m_changed + +import debug # pyflakes:ignore + + +def filter_from_queryargs(request): + #@debug.trace + def fix_ranges(d): + for k,v in d.items(): + if v.startswith("[") and v.endswith("]"): + d[k] = [ s for s in v[1:-1].split(",") if s ] + elif "," in v: + d[k] = [ s for s in v.split(",") if s ] + if k.endswith('__in') and not isinstance(d[k], list): + d[k] = [ d[k] ] + return d + def is_ascii(s): + return all(ord(c) < 128 for c in s) + # limit parameter keys to ascii. + params = dict( (k,v) for (k,v) in request.GET.items() if is_ascii(k) ) + filter = fix_ranges(dict([(k,params[k]) for k in params.keys() if not k.startswith("not__")])) + exclude = fix_ranges(dict([(k[5:],params[k]) for k in params.keys() if k.startswith("not__")])) + return filter, exclude + +def unique_obj_name(obj): + """Return a unique string representation for an object, based on app, class and ID + """ + app = obj._meta.app_label + model = obj.__class__.__name__.lower() + id = obj.pk + return "%s.%s[%s]" % (app,model,id) + +def cached_get(key, calculate_value, timeout=None): + """Try to get value from cache using key. If no value exists calculate + it by calling calculate_value. Timeout is defined in seconds.""" + value = cache.get(key) + if value is None: + value = calculate_value() + cache.set(key, value, timeout) + return value + +def model_top_level_cache_key(model): + return model.__module__ + '.' + model._meta.model.__name__ + +def clear_top_level_cache(sender, instance, *args, **kwargs): + cache.delete(model_top_level_cache_key(instance)) + +def clear_top_level_cache_m2m(sender, instance, action, reverse, model, *args, **kwargs): + # Purge cache for both models affected and the potentially custom 'through' model + cache.delete_many(( + model_top_level_cache_key(instance), + model_top_level_cache_key(model), + model_top_level_cache_key(sender), + )) + +post_save.connect(clear_top_level_cache, dispatch_uid='clear_top_level_cache') +post_delete.connect(clear_top_level_cache, dispatch_uid='clear_top_level_cache') +m2m_changed.connect(clear_top_level_cache_m2m, dispatch_uid='clear_top_level_cache') + +class AdminJsonSerializer(Serializer): + """ + Serializes a QuerySet to Json, with selectable object expansion. + The representation is different from that of the builtin Json + serializer in that there is no separate "model", "pk" and "fields" + entries for each object, instead only the "fields" dictionary is + serialized, and the model is the key of a top-level dictionary + entry which encloses the table serialization: + { + "app.model": { + "1": { + "foo": "1", + "bar": 42, + } + } + } + """ + + internal_use_only = False + use_natural_keys = False + + def serialize(self, queryset, **options): + qi = options.get('query_info', '') + if len(list(queryset)) == 1: + obj = queryset[0] + key = 'json:%s:%s' % (hashlib.md5(qi).hexdigest(), unique_obj_name(obj)) + is_cached = cache.get(model_top_level_cache_key(obj)) is True + if is_cached: + value = cached_get(key, lambda: super(AdminJsonSerializer, self).serialize(queryset, **options)) + else: + value = super(AdminJsonSerializer, self).serialize(queryset, **options) + cache.set(key, value) + cache.set(model_top_level_cache_key(obj), True) + return value + else: + return super(AdminJsonSerializer, self).serialize(queryset, **options) + + def start_serialization(self): + super(AdminJsonSerializer, self).start_serialization() + self.json_kwargs.pop("expand", None) + self.json_kwargs.pop("query_info", None) + + def get_dump_object(self, obj): + return self._current + + def end_object(self, obj): + expansions = [ n.split("__")[0] for n in self.options.get('expand', []) if n ] + for name in expansions: + try: + field = getattr(obj, name) + #self._current["_"+name] = smart_text(field) + if not isinstance(field, Field): + options = self.options.copy() + options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ] + if hasattr(field, "all"): + if options["expand"]: + # If the following code (doing qs.select_related() is commented out it + # is because it has the unfortunate side effect of changing the json + # rendering of booleans, from 'true/false' to '1/0', but only for the + # models pulled in by select_related(). If that's acceptable, we can + # comment this in again later. (The problem is known, captured in + # Django issue #15040: https://code.djangoproject.com/ticket/15040 + self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all().select_related() ]) + # self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all() ]) + else: + self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field.all() ]) + else: + if callable(field): + try: + field_value = field() + except Exception: + field_value = None + else: + field_value = field + if isinstance(field_value, QuerySet) or isinstance(field_value, list): + self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ]) + else: + if hasattr(field_value, "_meta"): + self._current[name] = self.expand_related(field_value, name) + else: + self._current[name] = unicode(field_value) + except ObjectDoesNotExist: + pass + except AttributeError: + names = [f.name for f in obj._meta.get_fields()] + if name in names and hasattr(obj, '%s_set' % name): + related_objects = getattr(obj, '%s_set' % name).all() + if self.options["expand"]: + self._current[name] = dict([(rel.pk, self.expand_related(rel, name)) for rel in related_objects.select_related()]) + else: + self._current[name] = dict([(rel.pk, self.expand_related(rel, name)) for rel in related_objects]) + else: + raise FieldError("Cannot resolve keyword '%s' into field. " + "Choices are: %s" % (name, ", ".join(names))) + super(AdminJsonSerializer, self).end_object(obj) + + def expand_related(self, related, name): + options = self.options.copy() + options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ] + bytes = self.__class__().serialize([ related ], **options) + data = json.loads(bytes)[0] + if 'password' in data: + del data['password'] + return data + + def handle_fk_field(self, obj, field): + try: + related = getattr(obj, field.name) + except ObjectDoesNotExist: + related = None + if related is not None: + if field.name in self.options.get('expand', []): + related = self.expand_related(related, field.name) + elif self.use_natural_keys and hasattr(related, 'natural_key'): + related = related.natural_key() + elif field.remote_field.field_name == related._meta.pk.name: + # Related to remote object via primary key + related = smart_text(related._get_pk_val(), strings_only=True) + else: + # Related to remote object via other field + related = smart_text(getattr(related, field.remote_field.field_name), strings_only=True) + self._current[field.name] = related + + def handle_m2m_field(self, obj, field): + if field.remote_field.through._meta.auto_created: + if field.name in self.options.get('expand', []): + m2m_value = lambda value: self.expand_related(value, field.name) + elif self.use_natural_keys and hasattr(field.remote_field.to, 'natural_key'): + m2m_value = lambda value: value.natural_key() + else: + m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True) + self._current[field.name] = [m2m_value(related) + for related in getattr(obj, field.name).iterator()] + +class JsonExportMixin(object): + """ + Adds JSON export to a DetailView + """ + +# def json_object(self, request, object_id, extra_context=None): +# "The json view for an object of this model." +# try: +# obj = self.get_queryset().get(pk=unquote(object_id)) +# except self.model.DoesNotExist: +# # Don't raise Http404 just yet, because we haven't checked +# # permissions yet. We don't want an unauthenticated user to be able +# # to determine whether a given object exists. +# obj = None +# +# if obj is None: +# raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(self.model._meta.verbose_name), 'key': escape(object_id)}) +# +# content_type = 'application/json' +# return HttpResponse(serialize([ obj ], sort_keys=True, indent=3)[2:-2], content_type=content_type) + + def json_view(self, request, filter={}, expand=[]): + qfilter, exclude = filter_from_queryargs(request) + for k in qfilter.keys(): + if k.startswith("_"): + del qfilter[k] + qfilter.update(filter) + filter = qfilter + key = request.GET.get("_key", "pk") + exp = [ e for e in request.GET.get("_expand", "").split(",") if e ] + for e in exp: + while True: + expand.append(e) + if not "__" in e: + break + e = e.rsplit("__", 1)[0] + # + expand = set(expand) + content_type = 'application/json' + query_info = "%s?%s" % (request.META["PATH_INFO"], request.META["QUERY_STRING"]) + try: + qs = self.get_queryset().filter(**filter).exclude(**exclude) + except (FieldError, ValueError) as e: + return HttpResponse(json.dumps({u"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) + try: + if expand: + qs = qs.select_related() + serializer = AdminJsonSerializer() + items = [(getattr(o, key), serializer.serialize([o], expand=expand, query_info=query_info) ) for o in qs ] + qd = dict( ( k, json.loads(v)[0] ) for k,v in items ) + except (FieldError, ValueError) as e: + return HttpResponse(json.dumps({u"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) + text = json.dumps({smart_text(self.model._meta): qd}, sort_keys=True, indent=3) + return HttpResponse(text, content_type=content_type) + diff --git a/ietf/api/tests.py b/ietf/api/tests.py index a88ab3648..4cd66c564 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -20,8 +20,9 @@ import debug # pyflakes:ignore from ietf.group.factories import RoleFactory from ietf.meeting.factories import MeetingFactory, SessionFactory from ietf.meeting.test_data import make_meeting_test_data +from ietf.person.factories import PersonFactory from ietf.person.models import PersonalApiKey -from ietf.utils.test_utils import TestCase +from ietf.utils.test_utils import TestCase, login_testing_unauthorized OMITTED_APPS = ( 'ietf.secr.meetings', @@ -124,6 +125,17 @@ class CustomApiTestCase(TestCase): self.assertEqual(event.by, recman) + def test_person_export(self): + person = PersonFactory() + url = urlreverse('ietf.api.views.PersonExportView') + login_testing_unauthorized(self, person.user.username, url) + r = self.client.get(url) + jsondata = r.json() + data = jsondata['person.person'][str(person.id)] + self.assertEqual(data['name'], person.name) + self.assertEqual(data['ascii'], person.ascii) + self.assertEqual(data['user']['email'], person.user.email) + class TastypieApiTestCase(ResourceTestCaseMixin, TestCase): def __init__(self, *args, **kwargs): self.apps = {} diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 8f29e2a5e..8751fb8c9 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -20,6 +20,7 @@ urlpatterns = [ url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url), url(r'^submit/?$', submit_views.api_submit), url(r'^iesg/position', views_ballot.api_set_position), + url(r'^export/person/$', api_views.PersonExportView.as_view()), ] # Additional (standard) Tastypie endpoints diff --git a/ietf/api/views.py b/ietf/api/views.py index 74d453513..ceb8f4f58 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -6,17 +6,24 @@ from __future__ import unicode_literals from jwcrypto.jwk import JWK from django.conf import settings +from django.contrib.auth.decorators import login_required from django.http import HttpResponse -from django.shortcuts import render +from django.shortcuts import render, get_object_or_404 from django.urls import reverse +from django.utils.decorators import method_decorator +from django.views.decorators.gzip import gzip_page +from django.views.generic.detail import DetailView from tastypie.exceptions import BadRequest from tastypie.utils.mime import determine_format, build_content_type from tastypie.utils import is_valid_jsonp_callback_value +from tastypie.serializers import Serializer import debug # pyflakes:ignore -from ietf.api import Serializer, _api_list +from ietf.person.models import Person +from ietf.api import _api_list +from ietf.api.serializer import JsonExportMixin def top_level(request): available_resources = {} @@ -50,3 +57,19 @@ def api_help(request): key.import_from_pem(settings.API_PUBLIC_KEY_PEM) return render(request, "api/index.html", {'key': key, 'settings':settings, }) + +@method_decorator((login_required, gzip_page), name='dispatch') +class PersonExportView(DetailView, JsonExportMixin): + model = Person + + def get(self, request): + person = get_object_or_404(self.model, user=request.user) + expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', + 'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase', + 'iprevent', 'liaisonstatementevent', 'whitelisted', 'schedule', 'constraint', 'session', 'message', + 'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent', + 'reviewersettings', 'reviewsecretarysettings', 'unavailableperiod', 'reviewwish', + 'nextreviewerinteam', 'reviewrequest', 'meetingregistration', 'submissionevent', 'preapproval', + 'user'] + return self.json_view(request, filter={'id':person.id}, expand=expand) + diff --git a/ietf/templates/registration/edit_profile.html b/ietf/templates/registration/edit_profile.html index 8c5079ce0..bf5b41f9d 100644 --- a/ietf/templates/registration/edit_profile.html +++ b/ietf/templates/registration/edit_profile.html @@ -33,6 +33,16 @@ if you wish to update or remove the information.

+

+ + All the information the datatracker has that is coupled to this account and visible + on this page or otherwise related to your work on ietf documents, is also available + to you as a JSON blob when + you are logged in. + +

+ +
@@ -126,6 +136,8 @@
+ + {% endblock %} {% block js %} diff --git a/release-coverage.json.gz b/release-coverage.json.gz index 41232eea8..28b5f0399 100644 Binary files a/release-coverage.json.gz and b/release-coverage.json.gz differ