altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

feat: get tool versions without VersionInfo model (#7418)

Browse source 
* feat: get tool versions without VersionInfo model

* chore: remove update_external_command_info call

* feat: get tool version without VersionInfo

* chore: Remove VersionInfo model

* chore: Migration to remove VersionInfo

* fix: handle errors better; ignore stderr

* fix: type annotation
This commit is contained in:
Jennifer Richards 2024-05-14 20:53:31 -03:00 committed by GitHub
parent 48e0aa23f5
commit a4e0354090
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 66 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
bin/daily
View file

@ -24,9 +24,6 @@ $DTDIR/bin/hourly
source $DTDIR/env/bin/activate
# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info
# Get IANA-registered yang models
#YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
# Hardcode the rsync target to avoid any unwanted deletes:

2
bin/dump-to-names-json
View file

@ -10,7 +10,7 @@
set -x
ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \
mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \
mailtrigger.MailTrigger mailtrigger.Recipient name \
group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \
| jq --sort-keys "sort_by(.model, .pk)" \
| jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json

44
ietf/name/fixtures/names.json
View file

@ -16789,49 +16789,5 @@
},
"model": "stats.countryalias",
"pk": 303
},
{
"fields": {
"command": "xym",
"switch": "--version",
"time": "2024-03-21T07:06:23.405Z",
"used": true,
"version": "xym 0.7.0"
},
"model": "utils.versioninfo",
"pk": 1
},
{
"fields": {
"command": "pyang",
"switch": "--version",
"time": "2024-03-21T07:06:23.755Z",
"used": true,
"version": "pyang 2.6.0"
},
"model": "utils.versioninfo",
"pk": 2
},
{
"fields": {
"command": "yanglint",
"switch": "--version",
"time": "2024-03-21T07:06:23.773Z",
"used": true,
"version": "yanglint SO 1.9.2"
},
"model": "utils.versioninfo",
"pk": 3
},
{
"fields": {
"command": "xml2rfc",
"switch": "--version",
"time": "2024-03-21T07:06:24.609Z",
"used": true,
"version": "xml2rfc 3.20.1"
},
"model": "utils.versioninfo",
"pk": 4
}
]

3
ietf/name/management/commands/generate_name_fixture.py
View file

@ -77,7 +77,6 @@ class Command(BaseCommand):
from ietf.mailtrigger.models import MailTrigger, Recipient
from ietf.meeting.models import BusinessConstraint
from ietf.stats.models import CountryAlias
from ietf.utils.models import VersionInfo
# Grab all ietf.name.models
for n in dir(ietf.name.models):
@ -87,7 +86,7 @@ class Command(BaseCommand):
model_objects[model_name(item)] = list(item.objects.all().order_by('pk'))
for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient,
CountryAlias, VersionInfo, BusinessConstraint ):
CountryAlias, BusinessConstraint ):
model_objects[model_name(m)] = list(m.objects.all().order_by('pk'))
for m in ( DBTemplate, ):

24
ietf/submit/checkers.py
View file

@ -14,8 +14,8 @@ from django.conf import settings
import debug # pyflakes:ignore
from ietf.utils import tool_version
from ietf.utils.log import log, assertion
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
from ietf.utils.test_runner import set_coverage_checking
@ -177,8 +177,10 @@ class DraftYangChecker(object):
model_list = list(set(model_list))
command = "xym"
cmd_version = VersionInfo.objects.get(command=command).version
message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err)
message = "{version}:\n{output}\n\n".format(
version=tool_version[command],
output=out.replace('\n\n', '\n').strip() if code == 0 else err,
)
results.append({
"name": name,
@ -209,7 +211,6 @@ class DraftYangChecker(object):
# pyang
cmd_template = settings.SUBMIT_PYANG_COMMAND
command = [ w for w in cmd_template.split() if not '=' in w ][0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(libs=modpath, model=path)
venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env')
venv_bin = os.path.join(venv_path, 'bin')
@ -238,14 +239,17 @@ class DraftYangChecker(object):
except ValueError:
pass
#passed = passed and code == 0 # For the submission tool. Yang checks always pass
message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err)
message += "{version}: {template}:\n{output}\n".format(
version=tool_version[command],
template=cmd_template,
output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err,
)
# yanglint
set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup
if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY):
cmd_template = settings.SUBMIT_YANGLINT_COMMAND
command = [ w for w in cmd_template.split() if not '=' in w ][0]
cmd_version = VersionInfo.objects.get(command=command).version
cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir,
draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR,
cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, )
@ -264,7 +268,11 @@ class DraftYangChecker(object):
except ValueError:
pass
#passed = passed and code == 0 # For the submission tool. Yang checks always pass
message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err)
message += "{version}: {template}:\n{output}\n".format(
version=tool_version[command],
template=cmd_template,
output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err,
)
set_coverage_checking(True)
else:
errors += 1
@ -293,4 +301,4 @@ class DraftYangChecker(object):
items = [ e for res in results for e in res["items"] ]
info['items'] = items
info['code']['yang'] = model_list
return passed, message, errors, warnings, info
return passed, message, errors, warnings, info

4
ietf/submit/tests.py
View file

@ -49,9 +49,9 @@ from ietf.submit.factories import SubmissionFactory, SubmissionExtResourceFactor
from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm
from ietf.submit.models import Submission, Preapproval, SubmissionExtResource
from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task
from ietf.utils import tool_version
from ietf.utils.accesstoken import generate_access_token
from ietf.utils.mail import outbox, get_payload_text
from ietf.utils.models import VersionInfo
from ietf.utils.test_utils import login_testing_unauthorized, TestCase
from ietf.utils.timezone import date_today
from ietf.utils.draft import PlaintextDraft
@ -1854,7 +1854,7 @@ class SubmitTests(BaseSubmitTestCase):
#
m = q('#yang-validation-message').text()
for command in ['xym', 'pyang', 'yanglint']:
version = VersionInfo.objects.get(command=command).version
version = tool_version[command]
if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)):
self.assertIn(version, m)
self.assertIn("draft-yang-testing-invalid-00.txt", m)

30
ietf/utils/__init__.py
View file

@ -1 +1,29 @@
# Copyright The IETF Trust 2007, All Rights Reserved
# Copyright The IETF Trust 2007-2024, All Rights Reserved
import subprocess
class _ToolVersionManager:
_known = [
"pyang",
"xml2rfc",
"xym",
"yanglint",
]
_versions: dict[str, str] = dict()
def __getitem__(self, item):
if item not in self._known:
return "Unknown"
elif item not in self._versions:
try:
self._versions[item] = subprocess.run(
[item, "--version"],
capture_output=True,
check=True,
).stdout.decode().strip()
except subprocess.CalledProcessError:
return "Unknown"
return self._versions[item]
tool_version = _ToolVersionManager()

7
ietf/utils/admin.py
View file

@ -5,8 +5,6 @@
from django.contrib import admin
from django.utils.encoding import force_str
from ietf.utils.models import VersionInfo
def name(obj):
if hasattr(obj, 'abbrev'):
return obj.abbrev()
@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin):
list_display = ['date', 'host', 'tz']
list_filter = ['date']
admin.site.register(DumpInfo, DumpInfoAdmin)
class VersionInfoAdmin(admin.ModelAdmin):
list_display = ['command', 'switch', 'version', 'time', ]
admin.site.register(VersionInfo, VersionInfoAdmin)

41
ietf/utils/management/commands/update_external_command_info.py
View file

@ -1,41 +0,0 @@
# Copyright The IETF Trust 2017-2020, All Rights Reserved
# -*- coding: utf-8 -*-
import sys
from textwrap import dedent
from django.core.management.base import BaseCommand
import debug # pyflakes:ignore
from ietf.utils.models import VersionInfo
from ietf.utils.pipe import pipe
class Command(BaseCommand):
"""
Update the version information for external commands used by the datatracker.
Iterates through the entries in the VersionInfo table, runs the relevant
command, and updates the version string with the result.
"""
help = dedent(__doc__).strip()
def handle(self, *filenames, **options):
for c in VersionInfo.objects.filter(used=True):
cmd = "%s %s" % (c.command, c.switch)
code, out, err = pipe(cmd)
out = out.decode('utf-8')
err = err.decode('utf-8')
if code != 0:
sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err))
else:
c.version = (out.strip()+'\n'+err.strip()).strip()
if options.get('verbosity', 1) > 1:
sys.stdout.write(
"Command: %s\n"
" Version: %s\n" % (cmd, c.version))
c.save()

16
ietf/utils/migrations/0002_delete_versioninfo.py Normal file
View file

@ -0,0 +1,16 @@
# Generated by Django 4.2.11 on 2024-05-03 21:03
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("utils", "0001_initial"),
]
operations = [
migrations.DeleteModel(
name="VersionInfo",
),
]

9
ietf/utils/models.py
View file

@ -9,15 +9,6 @@ class DumpInfo(models.Model):
host = models.CharField(max_length=128)
tz = models.CharField(max_length=32, default='UTC')
class VersionInfo(models.Model):
time = models.DateTimeField(auto_now=True)
command = models.CharField(max_length=32)
switch = models.CharField(max_length=16)
version = models.CharField(max_length=64)
used = models.BooleanField(default=True)
class Meta:
verbose_name_plural = 'VersionInfo'
class ForeignKey(models.ForeignKey):
"A local ForeignKey proxy which provides the on_delete value required under Django 2.0."
def __init__(self, to, on_delete=models.CASCADE, **kwargs):

20
ietf/utils/resources.py
View file

@ -12,7 +12,7 @@ from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from ietf import api
from ietf.utils.models import DumpInfo, VersionInfo
from ietf.utils.models import DumpInfo
class UserResource(ModelResource):
@ -43,21 +43,3 @@ class DumpInfoResource(ModelResource):
"host": ALL,
}
api.utils.register(DumpInfoResource())
class VersionInfoResource(ModelResource):
class Meta:
queryset = VersionInfo.objects.all()
serializer = api.Serializer()
cache = SimpleCache()
#resource_name = 'versioninfo'
ordering = ['id', ]
filtering = {
"id": ALL,
"time": ALL,
"command": ALL,
"switch": ALL,
"version": ALL,
"used": ALL,
}
api.utils.register(VersionInfoResource())