From 6089d4726c10256da100d5c3147cf0c051574e5b Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 11 May 2023 09:37:02 -0500 Subject: [PATCH 01/13] chore: remove debugging file write from test (#5598) --- ietf/meeting/tests_views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 8ecec0574..449ad03a3 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -355,8 +355,7 @@ class MeetingTests(BaseMeetingTestCase): # iCal, no session filtering ical_url = urlreverse("ietf.meeting.views.agenda_ical", kwargs=dict(num=meeting.number)) r = self.client.get(ical_url) - with open('./ical-output.ics', 'w') as f: - f.write(r.content.decode()) + assert_ical_response_is_valid(self, r) self.assertContains(r, "BEGIN:VTIMEZONE") self.assertContains(r, "END:VTIMEZONE") From 02f3606c004742991f502ba1df2e6121ddc08211 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 14:12:42 -0400 Subject: [PATCH 02/13] ci: fix skip tests build workflow --- .github/workflows/build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c353e9bc..86694f243 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,13 +253,24 @@ jobs: path: playwright/test-results/ if-no-files-found: ignore + tests-skip: + name: Quick Run + if: ${{ github.event.inputs.skiptests == 'true' }} + needs: [prepare] + runs-on: ubuntu-latest + + steps: + - name: Prepare for tests + run: | + echo "Quick run mode, skipping tests..." + # ----------------------------------------------------------------- # RELEASE # ----------------------------------------------------------------- release: name: Make Release if: ${{ success() }} - needs: [tests-python, tests-playwright, tests-playwright-legacy, prepare] + needs: [tests-python, tests-playwright, tests-playwright-legacy, tests-skip, prepare] runs-on: ubuntu-latest env: SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} From b767733f89b55a75dce5609a3a7741998536cfba Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 14:18:11 -0400 Subject: [PATCH 03/13] ci: fix skip tests build workflow (take 2) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86694f243..b7fb6946c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -269,7 +269,7 @@ jobs: # ----------------------------------------------------------------- release: name: Make Release - if: ${{ success() }} + if: ${{ !failure() && !cancelled() }} needs: [tests-python, tests-playwright, tests-playwright-legacy, tests-skip, prepare] runs-on: ubuntu-latest env: From 19ddfd6397b642b4ca4fd359b93886fe6277c3b3 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 14:33:48 -0400 Subject: [PATCH 04/13] ci: fix skip tests build workflow (take 3) --- .github/workflows/build.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7fb6946c..edf77228d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,17 +253,6 @@ jobs: path: playwright/test-results/ if-no-files-found: ignore - tests-skip: - name: Quick Run - if: ${{ github.event.inputs.skiptests == 'true' }} - needs: [prepare] - runs-on: ubuntu-latest - - steps: - - name: Prepare for tests - run: | - echo "Quick run mode, skipping tests..." - # ----------------------------------------------------------------- # RELEASE # ----------------------------------------------------------------- @@ -453,7 +442,7 @@ jobs: # ----------------------------------------------------------------- sandbox: name: Deploy to Sandbox - if: ${{ success() && github.event.inputs.sandbox == 'true' }} + if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} needs: [prepare, release] runs-on: [self-hosted, dev-server] env: From 492309b9fed14e322d524b1985d3a1dd8d506cca Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 14:38:42 -0400 Subject: [PATCH 05/13] ci: fix skip tests build workflow (take 4) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index edf77228d..8fd5f62ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -259,7 +259,7 @@ jobs: release: name: Make Release if: ${{ !failure() && !cancelled() }} - needs: [tests-python, tests-playwright, tests-playwright-legacy, tests-skip, prepare] + needs: [tests-python, tests-playwright, tests-playwright-legacy, prepare] runs-on: ubuntu-latest env: SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}} From 1ed87475d3b0443ed0e239eca006f6ff78d95512 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 15:15:20 -0400 Subject: [PATCH 06/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8fd5f62ad..4d99c3b6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,11 @@ on: default: true required: true type: boolean + legacySandbox: + description: 'Deploy to Legacy Sandbox' + default: false + required: false + type: boolean skiptests: description: 'Skip Tests' default: false @@ -474,3 +479,25 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | docker image prune -a -f + + legacySandbox: + name: Deploy to Legacy Sandbox + if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }} + needs: [prepare, release] + runs-on: [self-hosted, legacy-sandbox-server] + env: + PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + + steps: + - name: Download a Release Artifact + uses: actions/download-artifact@v3.0.2 + with: + name: release-${{ env.PKG_VERSION }} + path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} + +# - name: Deploy +# env: +# DEBIAN_FRONTEND: noninteractive +# run: | +# cp ../web/ietf/settings_local.py ietf/ + From a3459b438de9e7c50274ba66c4672bf9bc1d5b3e Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 15:36:50 -0400 Subject: [PATCH 07/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d99c3b6d..c569ec649 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,11 +493,27 @@ jobs: uses: actions/download-artifact@v3.0.2 with: name: release-${{ env.PKG_VERSION }} - path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} + path: /a/www/ietf-datatracker -# - name: Deploy -# env: -# DEBIAN_FRONTEND: noninteractive -# run: | -# cp ../web/ietf/settings_local.py ietf/ + - name: Extract Release + env: + DEBIAN_FRONTEND: noninteractive + working-directory: /a/www/ietf-datatracker + run: | + echo "Creating main.dev.${{ github.run_number }} directory..." + mkdir main.dev.${{ github.run_number }} + echo "Extracting release to main.dev.${{ github.run_number }} directory..." + tar xzf release.tar.gz -C a/www/ietf-datatracker/main.dev.${{ github.run_number }} + echo "Deleting release tarball..." + rm -rf release.tar.gz + - name: Setup Environment + env: + DEBIAN_FRONTEND: noninteractive + working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} + run: | + cp ../web/ietf/settings_local.py ietf/ + python3.9 -mvenv env + source env/bin/activate + pip install -r requirements.txt + pip freeze > frozen-requirements.txt From 6fe0c513d2fbb14dbf57d30c38f62bea3b75b1ce Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 15:44:45 -0400 Subject: [PATCH 08/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c569ec649..8a684efd2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -503,7 +503,7 @@ jobs: echo "Creating main.dev.${{ github.run_number }} directory..." mkdir main.dev.${{ github.run_number }} echo "Extracting release to main.dev.${{ github.run_number }} directory..." - tar xzf release.tar.gz -C a/www/ietf-datatracker/main.dev.${{ github.run_number }} + tar xzf release.tar.gz -C /a/www/ietf-datatracker/main.dev.${{ github.run_number }} echo "Deleting release tarball..." rm -rf release.tar.gz From 7a801c3c7521076ab9800b1040b9b6a0187ab6e2 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 16:13:32 -0400 Subject: [PATCH 09/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a684efd2..aeb751f33 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -512,8 +512,27 @@ jobs: DEBIAN_FRONTEND: noninteractive working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} run: | + echo "Copying settings from previous deploy..." cp ../web/ietf/settings_local.py ietf/ + echo "Installing Python dependencies..." python3.9 -mvenv env source env/bin/activate pip install -r requirements.txt pip freeze > frozen-requirements.txt + echo "Collecting static..." + ietf/manage.py collectstatic + echo "Running checks..." + ietf/manage.py check + + - name: Update Docker Containers + env: + DEBIAN_FRONTEND: noninteractive + working-directory: /a/docker/datatracker + run: | + echo "Pulling latest docker images..." + docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback + docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback + docker-compose pull + # echo "Shutting down containers..." + # docker-compose down -t 300 + From 94fad6edd4df7e896db0c5af0e37e0fa636d2f63 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 16:20:40 -0400 Subject: [PATCH 10/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aeb751f33..859eff66a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -493,17 +493,15 @@ jobs: uses: actions/download-artifact@v3.0.2 with: name: release-${{ env.PKG_VERSION }} - path: /a/www/ietf-datatracker + path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - name: Extract Release env: DEBIAN_FRONTEND: noninteractive - working-directory: /a/www/ietf-datatracker + working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} run: | - echo "Creating main.dev.${{ github.run_number }} directory..." - mkdir main.dev.${{ github.run_number }} - echo "Extracting release to main.dev.${{ github.run_number }} directory..." - tar xzf release.tar.gz -C /a/www/ietf-datatracker/main.dev.${{ github.run_number }} + echo "Extracting release tarball..." + tar xzf release.tar.gz echo "Deleting release tarball..." rm -rf release.tar.gz From 2590ab226c431420a40e4447c350eea6a9c3ce8e Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 11 May 2023 16:35:27 -0400 Subject: [PATCH 11/13] ci: build workflow - deploy to legacy sandbox (wip) --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 859eff66a..807eb3eb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -512,6 +512,7 @@ jobs: run: | echo "Copying settings from previous deploy..." cp ../web/ietf/settings_local.py ietf/ + rsync -a ../web/test/ test/ echo "Installing Python dependencies..." python3.9 -mvenv env source env/bin/activate From 1afdb0b7cc211e2739393427fb77490501047d1d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 11 May 2023 17:15:01 -0500 Subject: [PATCH 12/13] fix: show htmlized views of draft revisions without DocHistory (#5599) * fix: show htmlized views of draft revisions without DocHistory * fix: handle htmlized views of rfcs corectly --- ietf/doc/tests.py | 5 ++++- ietf/doc/views_doc.py | 34 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 13bffdc67..65859bf4b 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -617,7 +617,9 @@ Man Expires September 22, 2015 [Page 3] f.write(self.draft_text) def test_document_draft(self): - draft = WgDraftFactory(name='draft-ietf-mars-test',rev='01') + draft = WgDraftFactory(name='draft-ietf-mars-test',rev='01', create_revisions=range(0,2)) + + HolderIprDisclosureFactory(docs=[draft]) # Docs for testing relationships. Does not test 'possibly-replaces'. The 'replaced_by' direction @@ -785,6 +787,7 @@ Man Expires September 22, 2015 [Page 3] self.assertEqual(len(q('#sidebar option[value="draft-ietf-mars-test-00"][selected="selected"]')), 1) rfc = WgRfcFactory() + rfc.save_with_history([DocEventFactory(doc=rfc)]) (Path(settings.RFC_PATH) / rfc.get_base_name()).touch() r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=rfc.canonical_name()))) self.assertEqual(r.status_code, 200) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index e33a51f8b..ba7a3ee51 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -244,7 +244,7 @@ def document_main(request, name, rev=None, document_html=False): is_author = request.user.is_authenticated and doc.documentauthor_set.filter(person__user=request.user).exists() can_view_possibly_replaces = can_edit_replaces or is_author - rfc_number = name[3:] if name.startswith("") else None + rfc_number = name[3:] if name.startswith("rfc") else None draft_name = None for a in aliases: if a.startswith("draft"): @@ -487,13 +487,25 @@ def document_main(request, name, rev=None, document_html=False): html = None js = None css = None + diff_revisions = None + simple_diff_revisions = None if document_html: - html = doc.html_body() - if request.COOKIES.get("pagedeps") == "inline": - js = Path(finders.find("ietf/js/document_html.js")).read_text() - css = Path(finders.find("ietf/css/document_html_inline.css")).read_text() - if html: - css += Path(finders.find("ietf/css/document_html_txt.css")).read_text() + diff_revisions=get_diff_revisions(request, name, doc if isinstance(doc,Document) else doc.doc) + simple_diff_revisions = [t[1] for t in diff_revisions] + simple_diff_revisions.reverse() + if not doc.is_rfc() and rev != doc.rev: + # No DocHistory was found matching rev - snapshot will be false + # and doc will be a Document object, not a DocHistory + snapshot = True + doc = doc.fake_history_obj(rev) + else: + html = doc.html_body() + if request.COOKIES.get("pagedeps") == "inline": + js = Path(finders.find("ietf/js/document_html.js")).read_text() + css = Path(finders.find("ietf/css/document_html_inline.css")).read_text() + if html: + css += Path(finders.find("ietf/css/document_html_txt.css")).read_text() + return render(request, "doc/document_draft.html" if document_html is False else "doc/document_html.html", dict(doc=doc, document_html=document_html, @@ -505,7 +517,7 @@ def document_main(request, name, rev=None, document_html=False): name=name, content=content, split_content=split_content, - revisions=revisions, + revisions=simple_diff_revisions if document_html else revisions, snapshot=snapshot, stream_desc=stream_desc, latest_revision=latest_revision, @@ -544,7 +556,7 @@ def document_main(request, name, rev=None, document_html=False): status_changes=status_changes, proposed_status_changes=proposed_status_changes, rfc_aliases=rfc_aliases, - has_errata=doc.tags.filter(slug="errata"), + has_errata=doc.pk and doc.tags.filter(slug="errata"), # doc.pk == None if using a fake_history_obj published=published, file_urls=file_urls, additional_urls=additional_urls, @@ -569,7 +581,7 @@ def document_main(request, name, rev=None, document_html=False): review_assignments=review_assignments, no_review_from_teams=no_review_from_teams, due_date=due_date, - diff_revisions=get_diff_revisions(request, name, doc if isinstance(doc,Document) else doc.doc) if document_html else None + diff_revisions=diff_revisions )) if doc.type_id == "charter": @@ -859,7 +871,7 @@ def document_html(request, name, rev=None): if not os.path.exists(doc.get_file_name()): raise Http404("File not found: %s" % doc.get_file_name()) - return document_main(request, name=doc.name, rev=doc.rev if not doc.is_rfc() else None, document_html=True) + return document_main(request, name=doc.canonical_name(), rev=doc.rev if not doc.is_rfc() else None, document_html=True) def document_pdfized(request, name, rev=None, ext=None): From 6b4115a52b04fae9bed83c442e8ee991dce7ae5a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 12 May 2023 09:03:13 -0500 Subject: [PATCH 13/13] test: ignore irrelevant difference in list element order (#5609) --- ietf/secr/sreq/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index c23c9b1ba..eb716e9d6 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -505,7 +505,7 @@ class SubmitRequestCase(TestCase): list(session.constraints().get(name='timerange').timeranges.all().values('name')), list(TimerangeName.objects.filter(name__in=['thursday-afternoon-early', 'thursday-afternoon-late']).values('name')) ) - self.assertEqual(list(session.joint_with_groups.all()), [group3, group4]) + self.assertEqual(set(list(session.joint_with_groups.all())), set([group3, group4])) def test_submit_request_invalid(self): MeetingFactory(type_id='ietf', date=date_today())