From 86997e1e95581354857ec037323df47a08eeeee0 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Tue, 3 Mar 2015 20:23:36 +0000 Subject: [PATCH] Turned the api.py file into a module. Moved the makeresources management command to the api module. Added some api tests. Added crawling of api files to the test-crawler. Adjusted some resource files discovered by the test suite and test-crawler. Removed a bunch of empty model files. - Legacy-Id: 9144 --- bin/test-crawl | 44 +++++++++++-- ietf/{api.py => api/__init__.py} | 0 .../management/commands/makeresources.py | 0 ietf/api/tests.py | 62 +++++++++++++++++++ ietf/idindex/models.py | 0 ietf/ietfauth/models.py | 1 - ietf/mailinglists/models.py | 2 - ietf/meeting/resources.py | 25 ++++++-- ietf/name/resources.py | 43 ++++++++++++- ietf/release/models.py | 1 - ietf/secr/announcement/models.py | 0 ietf/secr/areas/models.py | 1 - ietf/secr/console/models.py | 1 - ietf/secr/drafts/models.py | 1 - ietf/secr/groups/models.py | 1 - ietf/secr/roles/models.py | 1 - ietf/secr/rolodex/models.py | 1 - ietf/secr/sreq/models.py | 1 - ietf/secr/telechat/models.py | 1 - ietf/settings.py | 5 +- ietf/sync/models.py | 0 ietf/utils/models.py | 0 tastypie/serializers.py | 2 +- 23 files changed, 168 insertions(+), 25 deletions(-) rename ietf/{api.py => api/__init__.py} (100%) rename ietf/{utils => api}/management/commands/makeresources.py (100%) create mode 100644 ietf/api/tests.py delete mode 100644 ietf/idindex/models.py delete mode 100644 ietf/ietfauth/models.py delete mode 100644 ietf/mailinglists/models.py delete mode 100644 ietf/release/models.py delete mode 100644 ietf/secr/announcement/models.py delete mode 100644 ietf/secr/areas/models.py delete mode 100644 ietf/secr/console/models.py delete mode 100644 ietf/secr/drafts/models.py delete mode 100644 ietf/secr/groups/models.py delete mode 100644 ietf/secr/roles/models.py delete mode 100644 ietf/secr/rolodex/models.py delete mode 100644 ietf/secr/sreq/models.py delete mode 100644 ietf/secr/telechat/models.py delete mode 100644 ietf/sync/models.py delete mode 100644 ietf/utils/models.py diff --git a/bin/test-crawl b/bin/test-crawl index 11a79b02e..88dd89397 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -1,6 +1,6 @@ #!/usr/bin/env python -import os, sys, re, datetime, argparse, traceback, tempfile +import os, sys, re, datetime, argparse, traceback, tempfile, json # args parser = argparse.ArgumentParser( @@ -51,6 +51,7 @@ if args.url_file: if not initial_urls: initial_urls.append("/") + initial_urls.append("/api/v1") visited = set() urls = {} # url -> referrer @@ -78,8 +79,30 @@ def extract_html_urls(content): yield url +def extract_tastypie_urls(content): + VISIT_OBJECTS = False + VISIT_NEXT = False + data = json.loads(content) + for item in data: + if type(data[item]) is dict: + if "list_endpoint" in data[item]: + uri = data[item]["list_endpoint"] + yield uri + if VISIT_NEXT: + if "meta" in data and "next" in data["meta"]: + uri = data["meta"]["next"] + if uri != None: + yield uri + if VISIT_OBJECTS: + if "objects" in data: + object_list = data["objects"] + for i in range(len(object_list)): + if "resource_uri" in object_list[i]: + uri = object_list[i]["resource_uri"] + yield uri + django.setup() -client = django.test.Client() +client = django.test.Client(Accept='text/html,text/plain,application/json') for url in initial_urls: urls[url] = "[initial]" @@ -150,8 +173,19 @@ while urls: log("=============") log(traceback.format_exc()) log("=============") + elif ctype == "application/json": + try: + for u in extract_tastypie_urls(r.content): + if u not in visited and u not in urls: + urls[u] = url + referrers[u] = url + except: + log("error extracting urls from %s" % url) + log("=============") + log(traceback.format_exc()) + log("=============") else: - tags.append(u"FAIL (from %s)" % referrer) + tags.append(u"FAIL for %s\n (from %s)" % (url, referrer)) errors += 1 if elapsed.total_seconds() > slow_threshold: @@ -164,9 +198,9 @@ while urls: sec = acc_secs % 60 if (len(visited) % 100) == 1: - log("\nElapsed Visited Queue Code Time Url ... Notes") + log("\nElapsed Visited Queue Code Time Url ... Notes") - log("%2d:%02d:%02d %7d %6d %s %.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags))) + log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags))) logfile.close() sys.stderr.write("Output written to %s\n\n" % logfile.name) diff --git a/ietf/api.py b/ietf/api/__init__.py similarity index 100% rename from ietf/api.py rename to ietf/api/__init__.py diff --git a/ietf/utils/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py similarity index 100% rename from ietf/utils/management/commands/makeresources.py rename to ietf/api/management/commands/makeresources.py diff --git a/ietf/api/tests.py b/ietf/api/tests.py new file mode 100644 index 000000000..99cd42587 --- /dev/null +++ b/ietf/api/tests.py @@ -0,0 +1,62 @@ +import os +import sys +import json + +from django.test import Client +from django.conf import settings +from django.utils.importlib import import_module +from django.db import models + +from tastypie.test import ResourceTestCase + +import debug # pyflakes:ignore + +OMITTED_APPS = ( + 'ietf.secr.meetings', + 'ietf.secr.proceedings', + 'ietf.ipr', +) + +class TastypieApiTestCase(ResourceTestCase): + def __init__(self, *args, **kwargs): + self.apps = {} + for app_name in settings.INSTALLED_APPS: + if app_name.startswith('ietf') and not app_name in OMITTED_APPS: + app = import_module(app_name) + name = app_name.split('.',1)[-1] + models_path = os.path.join(os.path.dirname(app.__file__), "models.py") + if os.path.exists(models_path): + self.apps[name] = app + super(ResourceTestCase, self).__init__(*args, **kwargs) + + def test_api_top_level(self): + client = Client(Accept='application/json') + r = client.get("/api/v1/") + self.assertValidJSONResponse(r) + resource_list = json.loads(r.content) + + for name in self.apps: + if not name in self.apps: + sys.stderr.write("Expected a REST API resource for %s, but didn't find one\n" % name) + + for name in self.apps: + self.assertIn(name, resource_list, + "Expected a REST API resource for %s, but didn't find one" % name) + + def test_all_model_resources_exist(self): + client = Client(Accept='application/json') + r = client.get("/api/v1") + top = json.loads(r.content) + for name in self.apps: + app = self.apps[name] + self.assertEqual("/api/v1/%s/"%name, top[name]["list_endpoint"]) + r = client.get(top[name]["list_endpoint"]) + self.assertValidJSONResponse(r) + app_resources = json.loads(r.content) + model_list = models.get_models(app.models) + for model in model_list: + if not model._meta.model_name in app_resources.keys(): + #print("There doesn't seem to be any resource for model %s.models.%s"%(app.__name__,model.__name__,)) + self.assertIn(model._meta.model_name, app_resources.keys(), + "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) + diff --git a/ietf/idindex/models.py b/ietf/idindex/models.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ietf/ietfauth/models.py b/ietf/ietfauth/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/ietfauth/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/mailinglists/models.py b/ietf/mailinglists/models.py deleted file mode 100644 index a4b306690..000000000 --- a/ietf/mailinglists/models.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index 9e95ca123..d79f4aa45 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -5,8 +5,8 @@ from tastypie.constants import ALL, ALL_WITH_RELATIONS from ietf import api -from ietf.meeting.models import * # pyflakes:ignore - +from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, + TimeSlot, ScheduledSession, SessionPresentation ) from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): @@ -38,7 +38,7 @@ class ResourceAssociationResource(ModelResource): name = ToOneField(RoomResourceNameResource, 'name') class Meta: queryset = ResourceAssociation.objects.all() - #resource_name = 'resourceassociation' + resource_name = 'resourceassociation' filtering = { "id": ALL, "icon": ALL, @@ -144,7 +144,7 @@ class TimeSlotResource(ModelResource): type = ToOneField(TimeSlotTypeNameResource, 'type') location = ToOneField(RoomResource, 'location', null=True) sessions = ToManyField(SessionResource, 'sessions', null=True) - duration = TimedeltaField() + duration = api.TimedeltaField() class Meta: queryset = TimeSlot.objects.all() #resource_name = 'timeslot' @@ -183,3 +183,20 @@ class ScheduledSessionResource(ModelResource): } api.meeting.register(ScheduledSessionResource()) + + +from ietf.doc.resources import DocumentResource +class SessionPresentationResource(ModelResource): + session = ToOneField(SessionResource, 'session') + document = ToOneField(DocumentResource, 'document') + class Meta: + queryset = SessionPresentation.objects.all() + #resource_name = 'sessionpresentation' + filtering = { + "id": ALL, + "rev": ALL, + "session": ALL_WITH_RELATIONS, + "document": ALL_WITH_RELATIONS, + } +api.meeting.register(SessionPresentationResource()) + diff --git a/ietf/name/resources.py b/ietf/name/resources.py index 40d473423..3bb2c09d3 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -223,7 +223,7 @@ api.name.register(DocRelationshipNameResource()) class RoomResourceNameResource(ModelResource): class Meta: queryset = RoomResourceName.objects.all() - #resource_name = 'roomresourcename' + resource_name = 'roomresourcename' # Needed because tastypie otherwise removes 'resource' from the name filtering = { "slug": ALL, "name": ALL, @@ -299,3 +299,44 @@ class NomineePositionStateNameResource(ModelResource): } api.name.register(NomineePositionStateNameResource()) + + +class IprDisclosureStateNameResource(ModelResource): + class Meta: + queryset = IprDisclosureStateName.objects.all() + #resource_name = 'iprdisclosurestatename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(IprDisclosureStateNameResource()) + +class IprEventTypeNameResource(ModelResource): + class Meta: + queryset = IprEventTypeName.objects.all() + #resource_name = 'ipreventtypename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(IprEventTypeNameResource()) + +class IprLicenseTypeNameResource(ModelResource): + class Meta: + queryset = IprLicenseTypeName.objects.all() + #resource_name = 'iprlicensetypename' + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(IprLicenseTypeNameResource()) + diff --git a/ietf/release/models.py b/ietf/release/models.py deleted file mode 100644 index bce5b68f3..000000000 --- a/ietf/release/models.py +++ /dev/null @@ -1 +0,0 @@ -# This app has no models diff --git a/ietf/secr/announcement/models.py b/ietf/secr/announcement/models.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ietf/secr/areas/models.py b/ietf/secr/areas/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/areas/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/console/models.py b/ietf/secr/console/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/console/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/drafts/models.py b/ietf/secr/drafts/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/drafts/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/groups/models.py b/ietf/secr/groups/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/groups/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/roles/models.py b/ietf/secr/roles/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/roles/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/rolodex/models.py b/ietf/secr/rolodex/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/rolodex/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/sreq/models.py b/ietf/secr/sreq/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/sreq/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/secr/telechat/models.py b/ietf/secr/telechat/models.py deleted file mode 100644 index 8b1378917..000000000 --- a/ietf/secr/telechat/models.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ietf/settings.py b/ietf/settings.py index 983d97804..256b4752e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -201,6 +201,7 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'django.contrib.messages', 'tastypie', + 'ietf.api', 'ietf.person', 'ietf.name', 'ietf.group', @@ -270,10 +271,10 @@ TEST_BLUESHEET_DIR = "tmp-bluesheet-dir" TEST_CODE_COVERAGE_EXCLUDE = [ "*/tests*", - "*/0*", "*/admin.py", "*/migrations/*", - "*/test_runner.py" + "ietf/settings*", + "ietf/utils/test_runner.py", ] TEST_CODE_COVERAGE_MASTER_FILE = "coverage-master.json" TEST_CODE_COVERAGE_LATEST_FILE = "coverage-latest.json" diff --git a/ietf/sync/models.py b/ietf/sync/models.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/ietf/utils/models.py b/ietf/utils/models.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tastypie/serializers.py b/tastypie/serializers.py index 527ed3ef3..a5064a650 100644 --- a/tastypie/serializers.py +++ b/tastypie/serializers.py @@ -371,7 +371,7 @@ class Serializer(object): options = options or {} data = self.to_simple(data, options) - return djangojson.json.dumps(data, cls=djangojson.DjangoJSONEncoder, sort_keys=True, ensure_ascii=False) + return djangojson.json.dumps(data, cls=djangojson.DjangoJSONEncoder, sort_keys=True, ensure_ascii=False, indent=2) def from_json(self, content): """