altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

fix: keep the ftp dir in sync when the datatracker creates artifacts (#8401)

Browse source 
* chore: fix typo in log message

* fix: write new conflict reviews to the ftp directory

* fix: log exception description

* chore: avoid reusing a name for mypy

* fix: write non-meeting-related slides to ftp/slides

* fix: write status changes to ftp dir

* fix: copy draft yang modules to the ftp dir

the last commit above (see #8401) has no tests :(. There aren't tests of this yang scaffolding to add to. Adding a set of tests should be an independent larger effort.
This commit is contained in:
Robert Sparks 2025-01-09 11:23:20 -06:00 committed by GitHub
parent 8b7fa1199a
commit 7d464fcc55
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 95 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
ietf/doc/tests_conflict_review.py
View file

@ -4,6 +4,7 @@
import io
import os
from pathlib import Path
from pyquery import PyQuery
from textwrap import wrap
@ -387,7 +388,7 @@ class ConflictReviewTests(TestCase):
class ConflictReviewSubmitTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH']
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH','FTP_PATH']
def test_initial_submission(self):
doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
url = urlreverse('ietf.doc.views_conflict_review.submit',kwargs=dict(name=doc.name))
@ -403,9 +404,15 @@ class ConflictReviewSubmitTests(TestCase):
# Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
# sane post using textbox
path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.name, doc.rev))
basename = f"{doc.name}-{doc.rev}.txt"
path = Path(settings.CONFLICT_REVIEW_PATH) / basename
ftp_dir = Path(settings.FTP_DIR) / "conflict-reviews"
if not ftp_dir.exists():
ftp_dir.mkdir()
ftp_path = ftp_dir / basename
self.assertEqual(doc.rev,'00')
self.assertFalse(os.path.exists(path))
self.assertFalse(path.exists())
self.assertFalse(ftp_path.exists())
r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
self.assertEqual(r.status_code,302)
doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
@ -413,6 +420,7 @@ class ConflictReviewSubmitTests(TestCase):
with io.open(path) as f:
self.assertEqual(f.read(),"Some initial review text\n")
f.close()
self.assertTrue(ftp_path.exists())
self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
def test_subsequent_submission(self):

13
ietf/doc/tests_material.py
View file

@ -28,7 +28,7 @@ from ietf.utils.test_utils import TestCase, login_testing_unauthorized
class GroupMaterialTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH']
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH', 'FTP_DIR']
def setUp(self):
super().setUp()
self.materials_dir = self.tempdir("materials")
@ -37,6 +37,10 @@ class GroupMaterialTests(TestCase):
self.slides_dir.mkdir()
self.saved_document_path_pattern = settings.DOCUMENT_PATH_PATTERN
settings.DOCUMENT_PATH_PATTERN = self.materials_dir + "/{doc.type_id}/"
self.assertTrue(Path(settings.FTP_DIR).exists())
ftp_slides_dir = Path(settings.FTP_DIR) / "slides"
if not ftp_slides_dir.exists():
ftp_slides_dir.mkdir()
self.meeting_slides_dir = Path(settings.AGENDA_PATH) / "42" / "slides"
if not self.meeting_slides_dir.exists():
@ -112,7 +116,12 @@ class GroupMaterialTests(TestCase):
self.assertEqual(doc.title, "Test File - with fancy title")
self.assertEqual(doc.get_state_slug(), "active")
with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f:
basename=f"{doc.name}-{doc.rev}.pdf"
filepath=Path(self.materials_dir) / "slides" / basename
with filepath.open() as f:
self.assertEqual(f.read(), content)
ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename
with ftp_filepath.open() as f:
self.assertEqual(f.read(), content)
# check that posting same name is prevented

17
ietf/doc/tests_status_change.py
View file

@ -4,6 +4,7 @@
import io
import os
from pathlib import Path
import debug # pyflakes:ignore
@ -540,7 +541,7 @@ class StatusChangeTests(TestCase):
DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
class StatusChangeSubmitTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH']
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH', 'FTP_PATH']
def test_initial_submission(self):
doc = Document.objects.get(name='status-change-imaginary-mid-review')
url = urlreverse('ietf.doc.views_status_change.submit',kwargs=dict(name=doc.name))
@ -556,14 +557,19 @@ class StatusChangeSubmitTests(TestCase):
# Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox
# sane post using textbox
path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
self.assertEqual(doc.rev,'00')
self.assertFalse(os.path.exists(path))
basename = f"{doc.name}-{doc.rev}.txt"
filepath = Path(settings.STATUS_CHANGE_PATH) / basename
ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename
self.assertFalse(filepath.exists())
self.assertFalse(ftp_filepath.exists())
r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1"))
self.assertEqual(r.status_code,302)
doc = Document.objects.get(name='status-change-imaginary-mid-review')
self.assertEqual(doc.rev,'00')
with io.open(path) as f:
with filepath.open() as f:
self.assertEqual(f.read(),"Some initial review text\n")
with ftp_filepath.open() as f:
self.assertEqual(f.read(),"Some initial review text\n")
self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc)
@ -628,3 +634,6 @@ class StatusChangeSubmitTests(TestCase):
def setUp(self):
super().setUp()
DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org')
ftp_subdir=Path(settings.FTP_DIR)/"status-changes"
if not ftp_subdir.exists():
ftp_subdir.mkdir()

6
ietf/doc/utils_charter.py
View file

@ -112,10 +112,10 @@ def fix_charter_revision_after_approval(charter, by):
)
try:
os.link(new, ftp_filepath)
except IOError:
except IOError as ex:
log(
"There was an error creating a harlink at %s pointing to %s"
% (ftp_filepath, new)
"There was an error creating a hardlink at %s pointing to %s: %s"
% (ftp_filepath, new, ex)
)
events = []

14
ietf/doc/views_conflict_review.py
View file

@ -5,6 +5,7 @@
import datetime
import io
import os
from pathlib import Path
from django import forms
from django.shortcuts import render, get_object_or_404, redirect
@ -181,12 +182,21 @@ class UploadForm(forms.Form):
return get_cleaned_text_file_content(self.cleaned_data["txt"])
def save(self, review):
filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.name, review.rev))
with io.open(filename, 'w', encoding='utf-8') as destination:
basename = f"{review.name}-{review.rev}.txt"
filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename
with filepath.open('w', encoding='utf-8') as destination:
if self.cleaned_data['txt']:
destination.write(self.cleaned_data['txt'])
else:
destination.write(self.cleaned_data['content'])
ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename
try:
os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10
except IOError as e:
log.log(
"There was an error creating a hardlink at %s pointing to %s: %s"
% (ftp_filepath, filepath, e)
)
#This is very close to submit on charter - can we get better reuse?
@role_required('Area Director','Secretariat')

16
ietf/doc/views_material.py
View file

@ -3,8 +3,8 @@
# views for managing group materials (slides, ...)
import io
import os
from pathlib import Path
import re
from django import forms
@ -162,9 +162,21 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
f = form.cleaned_data["material"]
file_ext = os.path.splitext(f.name)[1]
with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest:
basename = f"{doc.name}-{doc.rev}{file_ext}" # Note the lack of a . before file_ext - see os.path.splitext
filepath = Path(doc.get_file_path()) / basename
with filepath.open('wb+') as dest:
for chunk in f.chunks():
dest.write(chunk)
if not doc.meeting_related():
log.assertion('doc.type_id == "slides"')
ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename
try:
os.link(filepath, ftp_filepath) # Path.hardlink is not available until 3.10
except IOError as ex:
log.log(
"There was an error creating a hardlink at %s pointing to %s: %s"
% (ftp_filepath, filepath, ex)
)
if prev_rev != doc.rev:
e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)

23
ietf/doc/views_status_change.py
View file

@ -5,6 +5,7 @@
import datetime
import io
import os
from pathlib import Path
import re
from typing import Dict # pyflakes:ignore
@ -33,6 +34,7 @@ from ietf.ietfauth.utils import has_role, role_required
from ietf.mailtrigger.utils import gather_address_lists
from ietf.name.models import DocRelationshipName, StdLevelName
from ietf.person.models import Person
from ietf.utils.log import log
from ietf.utils.mail import send_mail_preformatted
from ietf.utils.textupload import get_cleaned_text_file_content
from ietf.utils.timezone import date_today, DEADLINE_TZINFO
@ -154,12 +156,21 @@ class UploadForm(forms.Form):
return get_cleaned_text_file_content(self.cleaned_data["txt"])
def save(self, doc):
filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev))
with io.open(filename, 'w', encoding='utf-8') as destination:
if self.cleaned_data['txt']:
destination.write(self.cleaned_data['txt'])
else:
destination.write(self.cleaned_data['content'])
basename = f"{doc.name}-{doc.rev}.txt"
filename = Path(settings.STATUS_CHANGE_PATH) / basename
with io.open(filename, 'w', encoding='utf-8') as destination:
if self.cleaned_data['txt']:
destination.write(self.cleaned_data['txt'])
else:
destination.write(self.cleaned_data['content'])
try:
ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename
os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10
except IOError as ex:
log(
"There was an error creating a hardlink at %s pointing to %s: %s"
% (ftp_filename, filename, ex)
)
#This is very close to submit on charter - can we get better reuse?
@role_required('Area Director','Secretariat')

10
ietf/submit/checkers.py
View file

@ -4,6 +4,7 @@
import io
import os
from pathlib import Path
import re
import shutil
import sys
@ -280,6 +281,15 @@ class DraftYangChecker(object):
dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model)
shutil.move(path, dest)
ftp_dest = Path(settings.FTP_DIR) / "yang" / "draftmod" / model
try:
os.link(dest, ftp_dest)
except IOError as ex:
log(
"There was an error creating a hardlink at %s pointing to %s: %s"
% (ftp_dest, dest, ex)
)
# summary result
results.append({

4
ietf/submit/utils.py
View file

@ -8,6 +8,7 @@ import json
import os
import pathlib
import re
import subprocess
import sys
import time
import traceback
@ -1596,3 +1597,6 @@ def populate_yang_model_dirs():
modfile.unlink()
except UnicodeDecodeError as e:
log.log(f"Error processing {item.name}: {e}")
ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod"
subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir))