From a4e0354090945e9f38eda5a64a96810163d667d7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 14 May 2024 20:53:31 -0300 Subject: [PATCH] feat: get tool versions without VersionInfo model (#7418) * feat: get tool versions without VersionInfo model * chore: remove update_external_command_info call * feat: get tool version without VersionInfo * chore: Remove VersionInfo model * chore: Migration to remove VersionInfo * fix: handle errors better; ignore stderr * fix: type annotation --- bin/daily | 3 -- bin/dump-to-names-json | 2 +- ietf/name/fixtures/names.json | 44 ------------------- .../commands/generate_name_fixture.py | 3 +- ietf/submit/checkers.py | 24 ++++++---- ietf/submit/tests.py | 4 +- ietf/utils/__init__.py | 30 ++++++++++++- ietf/utils/admin.py | 7 --- .../commands/update_external_command_info.py | 41 ----------------- .../migrations/0002_delete_versioninfo.py | 16 +++++++ ietf/utils/models.py | 9 ---- ietf/utils/resources.py | 20 +-------- 12 files changed, 66 insertions(+), 137 deletions(-) delete mode 100644 ietf/utils/management/commands/update_external_command_info.py create mode 100644 ietf/utils/migrations/0002_delete_versioninfo.py diff --git a/bin/daily b/bin/daily index 8211e1e23..6557a8922 100755 --- a/bin/daily +++ b/bin/daily @@ -24,9 +24,6 @@ $DTDIR/bin/hourly source $DTDIR/env/bin/activate -# Update our information about the current version of some commands we use -$DTDIR/ietf/manage.py update_external_command_info - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: diff --git a/bin/dump-to-names-json b/bin/dump-to-names-json index 9c7dfac07..20d4e0f95 100644 --- a/bin/dump-to-names-json +++ b/bin/dump-to-names-json @@ -10,7 +10,7 @@ set -x ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \ - mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \ + mailtrigger.MailTrigger mailtrigger.Recipient name \ group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \ | jq --sort-keys "sort_by(.model, .pk)" \ | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 35679dcaa..913c6c987 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -16789,49 +16789,5 @@ }, "model": "stats.countryalias", "pk": 303 - }, - { - "fields": { - "command": "xym", - "switch": "--version", - "time": "2024-03-21T07:06:23.405Z", - "used": true, - "version": "xym 0.7.0" - }, - "model": "utils.versioninfo", - "pk": 1 - }, - { - "fields": { - "command": "pyang", - "switch": "--version", - "time": "2024-03-21T07:06:23.755Z", - "used": true, - "version": "pyang 2.6.0" - }, - "model": "utils.versioninfo", - "pk": 2 - }, - { - "fields": { - "command": "yanglint", - "switch": "--version", - "time": "2024-03-21T07:06:23.773Z", - "used": true, - "version": "yanglint SO 1.9.2" - }, - "model": "utils.versioninfo", - "pk": 3 - }, - { - "fields": { - "command": "xml2rfc", - "switch": "--version", - "time": "2024-03-21T07:06:24.609Z", - "used": true, - "version": "xml2rfc 3.20.1" - }, - "model": "utils.versioninfo", - "pk": 4 } ] diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index bbf33e600..ef30e54c7 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -77,7 +77,6 @@ class Command(BaseCommand): from ietf.mailtrigger.models import MailTrigger, Recipient from ietf.meeting.models import BusinessConstraint from ietf.stats.models import CountryAlias - from ietf.utils.models import VersionInfo # Grab all ietf.name.models for n in dir(ietf.name.models): @@ -87,7 +86,7 @@ class Command(BaseCommand): model_objects[model_name(item)] = list(item.objects.all().order_by('pk')) for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient, - CountryAlias, VersionInfo, BusinessConstraint ): + CountryAlias, BusinessConstraint ): model_objects[model_name(m)] = list(m.objects.all().order_by('pk')) for m in ( DBTemplate, ): diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 5822f155f..d29e2a235 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -14,8 +14,8 @@ from django.conf import settings import debug # pyflakes:ignore +from ietf.utils import tool_version from ietf.utils.log import log, assertion -from ietf.utils.models import VersionInfo from ietf.utils.pipe import pipe from ietf.utils.test_runner import set_coverage_checking @@ -177,8 +177,10 @@ class DraftYangChecker(object): model_list = list(set(model_list)) command = "xym" - cmd_version = VersionInfo.objects.get(command=command).version - message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + message = "{version}:\n{output}\n\n".format( + version=tool_version[command], + output=out.replace('\n\n', '\n').strip() if code == 0 else err, + ) results.append({ "name": name, @@ -209,7 +211,6 @@ class DraftYangChecker(object): # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env') venv_bin = os.path.join(venv_path, 'bin') @@ -238,14 +239,17 @@ class DraftYangChecker(object): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) # yanglint set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): cmd_template = settings.SUBMIT_YANGLINT_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) @@ -264,7 +268,11 @@ class DraftYangChecker(object): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) set_coverage_checking(True) else: errors += 1 @@ -293,4 +301,4 @@ class DraftYangChecker(object): items = [ e for res in results for e in res["items"] ] info['items'] = items info['code']['yang'] = model_list - return passed, message, errors, warnings, info \ No newline at end of file + return passed, message, errors, warnings, info diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 08b898c13..58a47aef8 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,9 +49,9 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text -from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -1854,7 +1854,7 @@ class SubmitTests(BaseSubmitTestCase): # m = q('#yang-validation-message').text() for command in ['xym', 'pyang', 'yanglint']: - version = VersionInfo.objects.get(command=command).version + version = tool_version[command] if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)): self.assertIn(version, m) self.assertIn("draft-yang-testing-invalid-00.txt", m) diff --git a/ietf/utils/__init__.py b/ietf/utils/__init__.py index 7f1df9760..fbe55eb04 100644 --- a/ietf/utils/__init__.py +++ b/ietf/utils/__init__.py @@ -1 +1,29 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2024, All Rights Reserved +import subprocess + + +class _ToolVersionManager: + _known = [ + "pyang", + "xml2rfc", + "xym", + "yanglint", + ] + _versions: dict[str, str] = dict() + + def __getitem__(self, item): + if item not in self._known: + return "Unknown" + elif item not in self._versions: + try: + self._versions[item] = subprocess.run( + [item, "--version"], + capture_output=True, + check=True, + ).stdout.decode().strip() + except subprocess.CalledProcessError: + return "Unknown" + return self._versions[item] + + +tool_version = _ToolVersionManager() diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index fa1ebb708..6c1c8726e 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -5,8 +5,6 @@ from django.contrib import admin from django.utils.encoding import force_str -from ietf.utils.models import VersionInfo - def name(obj): if hasattr(obj, 'abbrev'): return obj.abbrev() @@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] list_filter = ['date'] admin.site.register(DumpInfo, DumpInfoAdmin) - -class VersionInfoAdmin(admin.ModelAdmin): - list_display = ['command', 'switch', 'version', 'time', ] -admin.site.register(VersionInfo, VersionInfoAdmin) - diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py deleted file mode 100644 index e9e24f000..000000000 --- a/ietf/utils/management/commands/update_external_command_info.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import sys - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.utils.models import VersionInfo -from ietf.utils.pipe import pipe - -class Command(BaseCommand): - """ - Update the version information for external commands used by the datatracker. - - Iterates through the entries in the VersionInfo table, runs the relevant - command, and updates the version string with the result. - - """ - - help = dedent(__doc__).strip() - - def handle(self, *filenames, **options): - for c in VersionInfo.objects.filter(used=True): - cmd = "%s %s" % (c.command, c.switch) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code != 0: - sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err)) - else: - c.version = (out.strip()+'\n'+err.strip()).strip() - if options.get('verbosity', 1) > 1: - sys.stdout.write( - "Command: %s\n" - " Version: %s\n" % (cmd, c.version)) - c.save() diff --git a/ietf/utils/migrations/0002_delete_versioninfo.py b/ietf/utils/migrations/0002_delete_versioninfo.py new file mode 100644 index 000000000..2835bb017 --- /dev/null +++ b/ietf/utils/migrations/0002_delete_versioninfo.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.11 on 2024-05-03 21:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("utils", "0001_initial"), + ] + + operations = [ + migrations.DeleteModel( + name="VersionInfo", + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 0915537fd..21af5766e 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -9,15 +9,6 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') -class VersionInfo(models.Model): - time = models.DateTimeField(auto_now=True) - command = models.CharField(max_length=32) - switch = models.CharField(max_length=16) - version = models.CharField(max_length=64) - used = models.BooleanField(default=True) - class Meta: - verbose_name_plural = 'VersionInfo' - class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 6d61c5e2e..1252cfef1 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -12,7 +12,7 @@ from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo, VersionInfo +from ietf.utils.models import DumpInfo class UserResource(ModelResource): @@ -43,21 +43,3 @@ class DumpInfoResource(ModelResource): "host": ALL, } api.utils.register(DumpInfoResource()) - - -class VersionInfoResource(ModelResource): - class Meta: - queryset = VersionInfo.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'versioninfo' - ordering = ['id', ] - filtering = { - "id": ALL, - "time": ALL, - "command": ALL, - "switch": ALL, - "version": ALL, - "used": ALL, - } -api.utils.register(VersionInfoResource())