diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py index 485f5655e..d2f94922b 100644 --- a/ietf/doc/tests_conflict_review.py +++ b/ietf/doc/tests_conflict_review.py @@ -4,6 +4,7 @@ import io import os +from pathlib import Path from pyquery import PyQuery from textwrap import wrap @@ -387,7 +388,7 @@ class ConflictReviewTests(TestCase): class ConflictReviewSubmitTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CONFLICT_REVIEW_PATH','FTP_PATH'] def test_initial_submission(self): doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') url = urlreverse('ietf.doc.views_conflict_review.submit',kwargs=dict(name=doc.name)) @@ -403,9 +404,15 @@ class ConflictReviewSubmitTests(TestCase): # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox # sane post using textbox - path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.name, doc.rev)) + basename = f"{doc.name}-{doc.rev}.txt" + path = Path(settings.CONFLICT_REVIEW_PATH) / basename + ftp_dir = Path(settings.FTP_DIR) / "conflict-reviews" + if not ftp_dir.exists(): + ftp_dir.mkdir() + ftp_path = ftp_dir / basename self.assertEqual(doc.rev,'00') - self.assertFalse(os.path.exists(path)) + self.assertFalse(path.exists()) + self.assertFalse(ftp_path.exists()) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') @@ -413,6 +420,7 @@ class ConflictReviewSubmitTests(TestCase): with io.open(path) as f: self.assertEqual(f.read(),"Some initial review text\n") f.close() + self.assertTrue(ftp_path.exists()) self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc) def test_subsequent_submission(self): diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index 065ff09a9..aaea8fec3 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -28,7 +28,7 @@ from ietf.utils.test_utils import TestCase, login_testing_unauthorized class GroupMaterialTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH', 'FTP_DIR'] def setUp(self): super().setUp() self.materials_dir = self.tempdir("materials") @@ -37,6 +37,10 @@ class GroupMaterialTests(TestCase): self.slides_dir.mkdir() self.saved_document_path_pattern = settings.DOCUMENT_PATH_PATTERN settings.DOCUMENT_PATH_PATTERN = self.materials_dir + "/{doc.type_id}/" + self.assertTrue(Path(settings.FTP_DIR).exists()) + ftp_slides_dir = Path(settings.FTP_DIR) / "slides" + if not ftp_slides_dir.exists(): + ftp_slides_dir.mkdir() self.meeting_slides_dir = Path(settings.AGENDA_PATH) / "42" / "slides" if not self.meeting_slides_dir.exists(): @@ -112,7 +116,12 @@ class GroupMaterialTests(TestCase): self.assertEqual(doc.title, "Test File - with fancy title") self.assertEqual(doc.get_state_slug(), "active") - with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f: + basename=f"{doc.name}-{doc.rev}.pdf" + filepath=Path(self.materials_dir) / "slides" / basename + with filepath.open() as f: + self.assertEqual(f.read(), content) + ftp_filepath=Path(settings.FTP_DIR) / "slides" / basename + with ftp_filepath.open() as f: self.assertEqual(f.read(), content) # check that posting same name is prevented diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py index bec48ed4e..bd4da4c09 100644 --- a/ietf/doc/tests_status_change.py +++ b/ietf/doc/tests_status_change.py @@ -4,6 +4,7 @@ import io import os +from pathlib import Path import debug # pyflakes:ignore @@ -540,7 +541,7 @@ class StatusChangeTests(TestCase): DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org') class StatusChangeSubmitTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['STATUS_CHANGE_PATH', 'FTP_PATH'] def test_initial_submission(self): doc = Document.objects.get(name='status-change-imaginary-mid-review') url = urlreverse('ietf.doc.views_status_change.submit',kwargs=dict(name=doc.name)) @@ -556,14 +557,19 @@ class StatusChangeSubmitTests(TestCase): # Right now, nothing to test - we let people put whatever the web browser will let them put into that textbox # sane post using textbox - path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev)) self.assertEqual(doc.rev,'00') - self.assertFalse(os.path.exists(path)) + basename = f"{doc.name}-{doc.rev}.txt" + filepath = Path(settings.STATUS_CHANGE_PATH) / basename + ftp_filepath = Path(settings.FTP_DIR) / "status-changes" / basename + self.assertFalse(filepath.exists()) + self.assertFalse(ftp_filepath.exists()) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') self.assertEqual(doc.rev,'00') - with io.open(path) as f: + with filepath.open() as f: + self.assertEqual(f.read(),"Some initial review text\n") + with ftp_filepath.open() as f: self.assertEqual(f.read(),"Some initial review text\n") self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc) @@ -628,3 +634,6 @@ class StatusChangeSubmitTests(TestCase): def setUp(self): super().setUp() DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org') + ftp_subdir=Path(settings.FTP_DIR)/"status-changes" + if not ftp_subdir.exists(): + ftp_subdir.mkdir() diff --git a/ietf/doc/utils_charter.py b/ietf/doc/utils_charter.py index b29d1e303..287ce8cec 100644 --- a/ietf/doc/utils_charter.py +++ b/ietf/doc/utils_charter.py @@ -112,10 +112,10 @@ def fix_charter_revision_after_approval(charter, by): ) try: os.link(new, ftp_filepath) - except IOError: + except IOError as ex: log( - "There was an error creating a harlink at %s pointing to %s" - % (ftp_filepath, new) + "There was an error creating a hardlink at %s pointing to %s: %s" + % (ftp_filepath, new, ex) ) events = [] diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index ec5a18c7a..e55661ccd 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -5,6 +5,7 @@ import datetime import io import os +from pathlib import Path from django import forms from django.shortcuts import render, get_object_or_404, redirect @@ -181,12 +182,21 @@ class UploadForm(forms.Form): return get_cleaned_text_file_content(self.cleaned_data["txt"]) def save(self, review): - filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.name, review.rev)) - with io.open(filename, 'w', encoding='utf-8') as destination: + basename = f"{review.name}-{review.rev}.txt" + filepath = Path(settings.CONFLICT_REVIEW_PATH) / basename + with filepath.open('w', encoding='utf-8') as destination: if self.cleaned_data['txt']: destination.write(self.cleaned_data['txt']) else: destination.write(self.cleaned_data['content']) + ftp_filepath = Path(settings.FTP_DIR) / "conflict-reviews" / basename + try: + os.link(filepath, ftp_filepath) # Path.hardlink_to is not available until 3.10 + except IOError as e: + log.log( + "There was an error creating a hardlink at %s pointing to %s: %s" + % (ftp_filepath, filepath, e) + ) #This is very close to submit on charter - can we get better reuse? @role_required('Area Director','Secretariat') diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index b646ecf2f..361bf5f1e 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -3,8 +3,8 @@ # views for managing group materials (slides, ...) -import io import os +from pathlib import Path import re from django import forms @@ -162,9 +162,21 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): f = form.cleaned_data["material"] file_ext = os.path.splitext(f.name)[1] - with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest: + basename = f"{doc.name}-{doc.rev}{file_ext}" # Note the lack of a . before file_ext - see os.path.splitext + filepath = Path(doc.get_file_path()) / basename + with filepath.open('wb+') as dest: for chunk in f.chunks(): dest.write(chunk) + if not doc.meeting_related(): + log.assertion('doc.type_id == "slides"') + ftp_filepath = Path(settings.FTP_DIR) / doc.type_id / basename + try: + os.link(filepath, ftp_filepath) # Path.hardlink is not available until 3.10 + except IOError as ex: + log.log( + "There was an error creating a hardlink at %s pointing to %s: %s" + % (ftp_filepath, filepath, ex) + ) if prev_rev != doc.rev: e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev) diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 9034971eb..33b822348 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -5,6 +5,7 @@ import datetime import io import os +from pathlib import Path import re from typing import Dict # pyflakes:ignore @@ -33,6 +34,7 @@ from ietf.ietfauth.utils import has_role, role_required from ietf.mailtrigger.utils import gather_address_lists from ietf.name.models import DocRelationshipName, StdLevelName from ietf.person.models import Person +from ietf.utils.log import log from ietf.utils.mail import send_mail_preformatted from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils.timezone import date_today, DEADLINE_TZINFO @@ -154,12 +156,21 @@ class UploadForm(forms.Form): return get_cleaned_text_file_content(self.cleaned_data["txt"]) def save(self, doc): - filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.name, doc.rev)) - with io.open(filename, 'w', encoding='utf-8') as destination: - if self.cleaned_data['txt']: - destination.write(self.cleaned_data['txt']) - else: - destination.write(self.cleaned_data['content']) + basename = f"{doc.name}-{doc.rev}.txt" + filename = Path(settings.STATUS_CHANGE_PATH) / basename + with io.open(filename, 'w', encoding='utf-8') as destination: + if self.cleaned_data['txt']: + destination.write(self.cleaned_data['txt']) + else: + destination.write(self.cleaned_data['content']) + try: + ftp_filename = Path(settings.FTP_DIR) / "status-changes" / basename + os.link(filename, ftp_filename) # Path.hardlink is not available until 3.10 + except IOError as ex: + log( + "There was an error creating a hardlink at %s pointing to %s: %s" + % (ftp_filename, filename, ex) + ) #This is very close to submit on charter - can we get better reuse? @role_required('Area Director','Secretariat') diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index d29e2a235..89908748a 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -4,6 +4,7 @@ import io import os +from pathlib import Path import re import shutil import sys @@ -280,6 +281,15 @@ class DraftYangChecker(object): dest = os.path.join(settings.SUBMIT_YANG_DRAFT_MODEL_DIR, model) shutil.move(path, dest) + ftp_dest = Path(settings.FTP_DIR) / "yang" / "draftmod" / model + try: + os.link(dest, ftp_dest) + except IOError as ex: + log( + "There was an error creating a hardlink at %s pointing to %s: %s" + % (ftp_dest, dest, ex) + ) + # summary result results.append({ diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index 590678504..16cccc9b5 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -8,6 +8,7 @@ import json import os import pathlib import re +import subprocess import sys import time import traceback @@ -1596,3 +1597,6 @@ def populate_yang_model_dirs(): modfile.unlink() except UnicodeDecodeError as e: log.log(f"Error processing {item.name}: {e}") + + ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod" + subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir))