From 4c297ba4c05843dde11aaabaf12304ff460307c0 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Mon, 26 Mar 2018 21:26:20 +0000 Subject: [PATCH] Changed settings so that cache keys used by memcached includes the release version, in order to avoid stale and incorrect cache content on new release deployment. Made it easier to pick out cache key composition by normalizing the variable name. Adjusted cache prefix and the composition of some keys to have a unique and consistent cache key prefix. - Legacy-Id: 14943 --- ietf/checks.py | 12 ++++++------ ietf/doc/models.py | 6 +++--- ietf/doc/views_search.py | 10 +++++----- ietf/doc/views_stats.py | 6 +++--- ietf/release/views.py | 13 ++++++------- ietf/settings.py | 12 +++++++++--- ietf/stats/views.py | 4 ++-- 7 files changed, 34 insertions(+), 29 deletions(-) diff --git a/ietf/checks.py b/ietf/checks.py index 4697195ec..963b37f16 100644 --- a/ietf/checks.py +++ b/ietf/checks.py @@ -272,18 +272,18 @@ def check_cache(app_configs, **kwargs): hint = "Please check that the configured cache backend is available.\n", id = "datatracker.%s" % errnum, ) - key = "ietf:checks:check_cache" + cache_key = "checks:check_cache" val = os.urandom(32) wait = 1 - cache.set(key, val, wait) - if not cache.get(key) == val: + cache.set(cache_key, val, wait) + if not cache.get(cache_key) == val: errors.append(cache_error("Could not get value from cache", "E0014")) time.sleep(wait+1) # should have timed out - if cache.get(key) == val: + if cache.get(cache_key) == val: errors.append(cache_error("Cache value didn't time out", "E0015")) - cache.set(key, val, settings.SESSION_COOKIE_AGE) - if not cache.get(key) == val: + cache.set(cache_key, val, settings.SESSION_COOKIE_AGE) + if not cache.get(cache_key) == val: errors.append(cache_error("Cache didn't accept session cookie age", "E0016")) return errors diff --git a/ietf/doc/models.py b/ietf/doc/models.py index e2c156e0c..654eb5050 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -468,14 +468,14 @@ class DocumentInfo(models.Model): html = "" if text: cache = caches['htmlized'] - key = name.split('.')[0] - html = cache.get(key) + cache_key = name.split('.')[0] + html = cache.get(cache_key) if not html: # The path here has to match the urlpattern for htmlized # documents in order to produce correct intra-document links html = rfc2html.markup(text, path=settings.HTMLIZER_URL_PREFIX) if html: - cache.set(key, html, settings.HTMLIZER_CACHE_TIME) + cache.set(cache_key, html, settings.HTMLIZER_CACHE_TIME) return html class Meta: diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 69285011f..8a1c12a21 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -204,11 +204,11 @@ def search(request): if not form.is_valid(): return HttpResponseBadRequest("form not valid: %s" % form.errors) - key = get_search_cache_key(get_params) - results = cache.get(key) + cache_key = get_search_cache_key(get_params) + results = cache.get(cache_key) if not results: results = retrieve_search_results(form) - cache.set(key, results) + cache.set(cache_key, results) results, meta = prepare_document_table(request, results, get_params) meta['searching'] = True @@ -242,8 +242,8 @@ def search_for_name(request, name): return None - def cached_redirect(key, url): - cache.set(key, url, settings.CACHE_MIDDLEWARE_SECONDS) + def cached_redirect(cache_key, url): + cache.set(cache_key, url, settings.CACHE_MIDDLEWARE_SECONDS) return HttpResponseRedirect(url) n = name diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 54249774b..ba9b0d163 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -124,15 +124,15 @@ def chart_newrevisiondocevent(request): def chart_data_newrevisiondocevent(request): queryargs = request.GET if queryargs: - key = get_search_cache_key(queryargs) - results = cache.get(key) + cache_key = get_search_cache_key(queryargs) + results = cache.get(cache_key) if not results: form = SearchForm(queryargs) if not form.is_valid(): return HttpResponseBadRequest("form not valid: %s" % form.errors) results = retrieve_search_results(form) if results.exists(): - cache.set(key, results) + cache.set(cache_key, results) if results.exists(): data = model_to_timeline_data(DocEvent, doc__in=results, type='new_revision') else: diff --git a/ietf/release/views.py b/ietf/release/views.py index 4259d25ee..faf560c64 100644 --- a/ietf/release/views.py +++ b/ietf/release/views.py @@ -24,7 +24,6 @@ import debug # pyflakes:ignore import time time.strptime('1984', '%Y') # we do this to force lib loading, instead of it happening lazily when changelog calls tzparse later -import ietf def trac_links(text): # changeset links @@ -35,8 +34,8 @@ def trac_links(text): def get_coverage_data(): - key = 'ietf:release:get_coverage_data:%s' % ietf.__version__ - coverage_data = cache.get(key) + cache_key = 'release:get_coverage_data' + coverage_data = cache.get(cache_key) if not coverage_data: coverage_data = {} if os.path.exists(settings.TEST_COVERAGE_MASTER_FILE): @@ -46,16 +45,16 @@ def get_coverage_data(): else: with open(settings.TEST_COVERAGE_MASTER_FILE) as file: coverage_data = json.load(file) - cache.set(key, coverage_data, 60*60*24) + cache.set(cache_key, coverage_data, 60*60*24) return coverage_data def get_changelog_entries(): - key = 'ietf:release:get_changelog_entries:%s' % ietf.__version__ - log_entries = cache.get(key) + cache_key = 'release:get_changelog_entries' + log_entries = cache.get(cache_key) if not log_entries: if os.path.exists(settings.CHANGELOG_PATH): log_entries = changelog.parse(settings.CHANGELOG_PATH) - cache.set(key, log_entries, 60*60*24) + cache.set(cache_key, log_entries, 60*60*24) return log_entries def release(request, version=None): diff --git a/ietf/settings.py b/ietf/settings.py index 73d0e9196..ebd2528f8 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -616,6 +616,8 @@ CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211', + 'VERSION': __version__, + 'KEY_PREFIX': 'ietf', }, 'htmlized': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', @@ -1025,9 +1027,13 @@ if SERVER_MODE != 'production': TEMPLATES[0]['OPTIONS']['loaders'] = loaders CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - }, + 'default': { + #'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + #'LOCATION': '127.0.0.1:11211', + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'VERSION': __version__, + 'KEY_PREFIX': 'ietf:dt', + }, 'htmlized': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 6b0aafe1c..e90f4e68e 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -134,7 +134,7 @@ def document_stats(request, stats_type=None): return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) - cache_key = ("stats:document:%s:%s" % (stats_type, request.META.get('QUERY_STRING',''))) + cache_key = ("stats:document_stats:%s:%s" % (stats_type, request.META.get('QUERY_STRING',''))) data = cache.get(cache_key) if not data: names_limit = settings.STATS_NAMES_LIMIT @@ -766,7 +766,7 @@ def meeting_stats(request, num=None, stats_type=None): return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) - cache_key = "stats:meeting:%s:%s:%s" % (num, stats_type, request.META.get('QUERY_STRING','')) + cache_key = "stats:meeting_stats:%s:%s:%s" % (num, stats_type, request.META.get('QUERY_STRING','')) data = cache.get(cache_key) if not data: names_limit = settings.STATS_NAMES_LIMIT