diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 3995b4b49..609b641ef 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -14,11 +14,18 @@ jobs: publish: runs-on: ubuntu-latest permissions: - contents: read + contents: write packages: write steps: - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_COMMON_TOKEN }} + + - name: Set Version + run: | + printf -v CURDATE '%(%Y%m%dT%H%M)T' -1 + echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -42,4 +49,18 @@ jobs: file: docker/base.Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: ghcr.io/ietf-tools/datatracker-app-base:latest + tags: | + ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:latest + + - name: Update version references + run: | + sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE + + - name: Commit CHANGELOG.md + uses: stefanzweifel/git-auto-commit-action@v5 + with: + branch: main + commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' + file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 756600cc1..6041fe474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,7 @@ jobs: pkg_version: ${{ steps.buildvars.outputs.pkg_version }} from_tag: ${{ steps.semver.outputs.nextStrict }} to_tag: ${{ steps.semver.outputs.current }} + base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - uses: actions/checkout@v4 @@ -122,6 +123,11 @@ jobs: echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" fi + - name: Get Base Image Target Version + id: baseimgversion + run: | + echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT + # ----------------------------------------------------------------- # TESTS # ----------------------------------------------------------------- @@ -133,6 +139,7 @@ jobs: needs: [prepare] with: ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} + targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }} # ----------------------------------------------------------------- # RELEASE @@ -200,6 +207,7 @@ jobs: SKIP_TESTS: ${{ github.event.inputs.skiptests }} DEBIAN_FRONTEND: noninteractive BROWSERSLIST_IGNORE_OLD_DATA: 1 + TARGETBASE: ${{ needs.prepare.outputs.base_image_version }} with: host: ${{ steps.azlaunch.outputs.ipaddr }} port: 22 @@ -310,7 +318,7 @@ jobs: echo "==========================================================================" echo "Collecting statics..." echo "==========================================================================" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:latest sh dev/build/collectstatics.sh + sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:$TARGETBASE sh dev/build/collectstatics.sh echo "Pushing statics..." cd static aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64db03db0..4e0b31fb1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,12 +8,17 @@ on: default: false required: true type: boolean + targetBaseVersion: + description: 'Target Base Image Version' + default: latest + required: true + type: string jobs: tests-python: name: Python Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} services: db: @@ -43,7 +48,7 @@ jobs: exit 1 fi echo "Running tests..." - if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then + if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then echo "Lower coverage failures will be ignored." HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage else @@ -114,7 +119,7 @@ jobs: tests-playwright-legacy: name: Playwright Legacy Tests runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} strategy: fail-fast: false matrix: diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index bede86fac..728b37156 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:2024102800 +FROM ghcr.io/ietf-tools/datatracker-app-base:20241029T1632 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE new file mode 100644 index 000000000..2f8aac20f --- /dev/null +++ b/dev/build/TARGET_BASE @@ -0,0 +1 @@ +20241029T1632 diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 639e6ca85..077502db1 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -530,7 +530,7 @@ class DocumentInfo(models.Model): def replaced_by(self): return set([ r.document for r in self.related_that("replaces") ]) - def text(self): + def text(self, size = -1): path = self.get_file_name() root, ext = os.path.splitext(path) txtpath = root+'.txt' @@ -538,14 +538,21 @@ class DocumentInfo(models.Model): path = txtpath try: with io.open(path, 'rb') as file: - raw = file.read() + raw = file.read(size) except IOError: return None + text = None try: text = raw.decode('utf-8') except UnicodeDecodeError: - text = raw.decode('latin-1') - # + for back in range(1,4): + try: + text = raw[:-back].decode('utf-8') + break + except UnicodeDecodeError: + pass + if text is None: + text = raw.decode('latin-1') return text def text_or_error(self): diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 915dcebde..50c60aefc 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -84,7 +84,7 @@ from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc from ietf.utils import markup_txt, log, markdown -from ietf.utils.draft import PlaintextDraft +from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split @@ -2261,12 +2261,11 @@ def idnits2_state(request, name, rev=None): elif doc.intended_std_level: doc.deststatus = doc.intended_std_level.name else: - text = doc.text() + # 10000 is a conservative prefix on number of utf-8 encoded bytes to + # cover at least the first 10 lines of characters + text = doc.text(size=10000) if text: - parsed_draft = PlaintextDraft( - text=doc.text(), source=name, name_from_source=False - ) - doc.deststatus = parsed_draft.get_status() + doc.deststatus = get_status_from_draft_text(text) else: doc.deststatus = "Unknown" return render( diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index a1e79760e..50add5abb 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -131,6 +131,24 @@ def acronym_match(s, l): #_debug(" s:%s; l:%s => %s; %s" % (s, l, acronym, s==acronym)) return s == acronym +def get_status_from_draft_text(text): + + # Take prefix to shortcut work over very large drafts + # 5000 is conservatively much more than a full page of characters and we + # only want the first 10 lines. + text = text.strip()[:5000] # Take prefix to shortcut work over very large drafts + text = re.sub(".\x08", "", text) # Get rid of inkribbon backspace-emphasis + text = text.replace("\r\n", "\n") # Convert DOS to unix + text = text.replace("\r", "\n") # Convert MAC to unix + lines = text.split("\n")[:10] + status = None + for line in lines: + status_match = re.search(r"^\s*Intended [Ss]tatus:\s*(.*?) ", line) + if status_match: + status = status_match.group(1) + break + return status + class Draft: """Base class for drafts