diff --git a/.gitattributes b/.gitattributes index 937c0eb37..62f4aae43 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,280 @@ -/.yarn/releases/** binary -/.yarn/plugins/** binary +# Auto detect text files and perform LF normalization +* text=auto + +# --------------------------------------------------- +# Python Projects +# --------------------------------------------------- + +# Source files +*.pxd text diff=python +*.py text diff=python +*.py3 text diff=python +*.pyw text diff=python +*.pyx text diff=python +*.pyz text diff=python +*.pyi text diff=python + +# Binary files +*.db binary +*.p binary +*.pkl binary +*.pickle binary +*.pyc binary export-ignore +*.pyo binary export-ignore +*.pyd binary + +# Jupyter notebook +*.ipynb text eol=lf + +# --------------------------------------------------- +# Web Projects +# --------------------------------------------------- + +# Source code +*.bash text eol=lf +*.bat text eol=crlf +*.cmd text eol=crlf +*.coffee text +*.css text diff=css +*.htm text diff=html +*.html text diff=html +*.inc text +*.ini text +*.js text +*.mjs text +*.cjs text +*.json text +*.jsx text +*.less text +*.ls text +*.map text -diff +*.od text +*.onlydata text +*.php text diff=php +*.pl text +*.ps1 text eol=crlf +*.py text diff=python +*.rb text diff=ruby +*.sass text +*.scm text +*.scss text diff=css +*.sh text eol=lf +.husky/* text eol=lf +*.sql text +*.styl text +*.tag text +*.ts text +*.tsx text +*.xml text +*.xhtml text diff=html + +# Docker +Dockerfile text + +# Documentation +*.ipynb text eol=lf +*.markdown text diff=markdown +*.md text diff=markdown +*.mdwn text diff=markdown +*.mdown text diff=markdown +*.mkd text diff=markdown +*.mkdn text diff=markdown +*.mdtxt text +*.mdtext text +*.txt text +AUTHORS text +CHANGELOG text +CHANGES text +CONTRIBUTING text +COPYING text +copyright text +*COPYRIGHT* text +INSTALL text +license text +LICENSE text +NEWS text +readme text +*README* text +TODO text + +# Templates +*.dot text +*.ejs text +*.erb text +*.haml text +*.handlebars text +*.hbs text +*.hbt text +*.jade text +*.latte text +*.mustache text +*.njk text +*.phtml text +*.pug text +*.svelte text +*.tmpl text +*.tpl text +*.twig text +*.vue text + +# Configs +*.cnf text +*.conf text +*.config text +.editorconfig text +.env text +.gitattributes text +.gitconfig text +.htaccess text +*.lock text -diff +package.json text eol=lf +package-lock.json text eol=lf -diff +pnpm-lock.yaml text eol=lf -diff +.prettierrc text +yarn.lock text -diff +*.toml text +*.yaml text +*.yml text +browserslist text +Makefile text +makefile text +# Fixes syntax highlighting on GitHub to allow comments +tsconfig.json linguist-language=JSON-with-Comments + +# Heroku +Procfile text + +# Graphics +*.ai binary +*.bmp binary +*.eps binary +*.gif binary +*.gifv binary +*.ico binary +*.jng binary +*.jp2 binary +*.jpg binary +*.jpeg binary +*.jpx binary +*.jxr binary +*.pdf binary +*.png binary +*.psb binary +*.psd binary +*.svg text +*.svgz binary +*.tif binary +*.tiff binary +*.wbmp binary +*.webp binary + +# Audio +*.kar binary +*.m4a binary +*.mid binary +*.midi binary +*.mp3 binary +*.ogg binary +*.ra binary + +# Video +*.3gpp binary +*.3gp binary +*.as binary +*.asf binary +*.asx binary +*.avi binary +*.fla binary +*.flv binary +*.m4v binary +*.mng binary +*.mov binary +*.mp4 binary +*.mpeg binary +*.mpg binary +*.ogv binary +*.swc binary +*.swf binary +*.webm binary + +# Archives +*.7z binary +*.gz binary +*.jar binary +*.rar binary +*.tar binary +*.zip binary + +# Fonts +*.ttf binary +*.eot binary +*.otf binary +*.woff binary +*.woff2 binary + +# Executables +*.exe binary +*.pyc binary +# Prevents massive diffs caused by vendored, minified files +**/.yarn/releases/** binary +**/.yarn/plugins/** binary + +# RC files (like .babelrc or .eslintrc) +*.*rc text + +# Ignore files (like .npmignore or .gitignore) +*.*ignore text + +# Prevents massive diffs from built files +dist/* binary + +# --------------------------------------------------- +# Common +# --------------------------------------------------- + +# Documents +*.bibtex text diff=bibtex +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain +*.md text diff=markdown +*.mdx text diff=markdown +*.tex text diff=tex +*.adoc text +*.textile text +*.mustache text +*.csv text eol=crlf +*.tab text +*.tsv text +*.txt text +*.sql text +*.epub diff=astextplain + +# Text files where line endings should be preserved +*.patch -text + +# --------------------------------------------------- +# Vzic specific +# --------------------------------------------------- + +*.pl text diff=perl +*.pm text diff=perl + +# C/C++ +*.c text diff=cpp +*.cc text diff=cpp +*.cxx text diff=cpp +*.cpp text diff=cpp +*.cpi text diff=cpp +*.c++ text diff=cpp +*.hpp text diff=cpp +*.h text diff=cpp +*.h++ text diff=cpp +*.hh text diff=cpp \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bf340c523..f1044223d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -137,6 +137,7 @@ jobs: uses: ./.github/workflows/tests.yml if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} needs: [prepare] + secrets: inherit with: ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }} skipSelenium: true @@ -149,7 +150,8 @@ jobs: name: Make Release if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: ubuntu-latest + runs-on: + group: hperf-8c32r permissions: contents: write packages: write @@ -166,215 +168,107 @@ jobs: fetch-depth: 1 fetch-tags: false - - name: Launch build VM - id: azlaunch - timeout-minutes: 10 - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }} - - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatracker \ - --name tmpGhaBuildVM-${{ github.run_number }} \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D8ads_v5 \ - --max-price -1 \ - --ephemeral-os-disk \ - --os-disk-size-gb 100 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --os-disk-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - - echo "Fetching SSH host public keys..." - until ssh-keyscan -t rsa $vminfo 2> /dev/null - do - echo "Will try again in 5 seconds..." - sleep 5 - done - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: 18.x - - name: Remote SSH into Build VM - uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Download a Coverage Results + if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: actions/download-artifact@v4.1.8 + with: + name: coverage + + - name: Make Release Build env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_SHA: ${{ github.sha }} - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_RUN_ID: ${{ github.run_id }} + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "PKG_VERSION: $PKG_VERSION" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ + yarn legacy:build + echo "Setting version $PKG_VERSION..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ env.SHOULD_DEPLOY == 'true' }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Release Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} AWS_DEFAULT_REGION: auto AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - PKG_VERSION: ${{ env.PKG_VERSION }} - SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }} - SKIP_TESTS: ${{ github.event.inputs.skiptests }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors + + - name: Augment dockerignore for docker image build + env: DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - TARGET_BASE: ${{ env.TARGET_BASE }} + run: | + cat >> .dockerignore < /dev/null \ - && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && sudo apt update \ - && sudo apt install gh -y - - echo "==========================================================================" - echo "Installing AWS CLI..." - echo "==========================================================================" - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install - - echo "==========================================================================" - echo "Install Node.js..." - echo "==========================================================================" - curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh - sudo bash nodesource_setup.sh - sudo apt-get install -y nodejs - sudo corepack enable - - echo "==========================================================================" - echo "Install Python 3.x..." - echo "==========================================================================" - sudo apt-get install python3 python3-dev -y - python3 --version - - echo "==========================================================================" - echo "Clone project..." - echo "==========================================================================" - sudo mkdir -p /workspace - sudo chown azureuser /workspace - cd /workspace - gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags - cd datatracker - - if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then - echo "==========================================================================" - echo "Downloading coverage..." - echo "==========================================================================" - gh run download $GITHUB_RUN_ID -n coverage - fi - - echo "==========================================================================" - echo "Building project..." - echo "==========================================================================" - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - if [ "$SHOULD_DEPLOY" = "true" ] ; then - echo "==========================================================================" - echo "Setting production flags in settings.py..." - echo "==========================================================================" - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - fi - - echo "==========================================================================" - echo "Build release tarball..." - echo "==========================================================================" - mkdir -p /workspace/release - tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt . - - echo "==========================================================================" - echo "Collecting statics..." - echo "==========================================================================" - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - cd .. - - echo "==========================================================================" - echo "Augment dockerignore for docker image build..." - echo "==========================================================================" - cat >> .dockerignore < by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", - "attachments": [ - { - "color": "28a745", - "fields": [ - { - "title": "Status", - "short": true, - "value": "Completed" - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Build by ${{ github.triggering_actor }}" + attachments: + - color: "28a745" + fields: + - title: "Status" + short: true + value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.27.0 + uses: slackapi/slack-github-action@v2 with: - channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + token: ${{ secrets.SLACK_GH_BOT }} + method: chat.postMessage payload: | - { - "text": "Datatracker Build by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>", - "attachments": [ - { - "color": "a82929", - "fields": [ - { - "title": "Status", - "short": true, - "value": "Failed" - } - ] - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} + channel: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} + text: "Datatracker Build by ${{ github.triggering_actor }}" + attachments: + - color: "a82929" + fields: + - title: "Status" + short: true + value: "Failed" # ----------------------------------------------------------------- # SANDBOX diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index 3f828430a..6d53a121a 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -38,7 +38,7 @@ jobs: ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into VM - uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1 + uses: appleboy/ssh-action@7eaf76671a0d7eec5d98ee897acda4f968735a17 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bb767513c..8eb6adc95 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,9 +73,11 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.6.0 + uses: codecov/codecov-action@v5 with: + disable_search: true files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} - name: Convert Coverage Results if: ${{ always() }} diff --git a/client/agenda/AgendaMobileBar.vue b/client/agenda/AgendaMobileBar.vue index 78af96e25..63611e21c 100644 --- a/client/agenda/AgendaMobileBar.vue +++ b/client/agenda/AgendaMobileBar.vue @@ -3,11 +3,13 @@ n-dropdown( :options='jumpToDayOptions' size='huge' + :show='isDropdownOpenRef' :show-arrow='true' trigger='click' @select='jumpToDay' + @clickoutside='handleCloseDropdown' ) - button + button(@click='handleOpenDropdown') i.bi.bi-arrow-down-circle button(@click='agendaStore.$patch({ filterShown: true })') i.bi.bi-funnel @@ -28,7 +30,7 @@ - -{% endblock %} -{% block title %}Document Statistics{% endblock %} -{% block content %} - {% origin %} -
-{% endblock %} \ No newline at end of file diff --git a/ietf/templates/group/group_leadership.html b/ietf/templates/group/group_leadership.html new file mode 100644 index 000000000..644be3e15 --- /dev/null +++ b/ietf/templates/group/group_leadership.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2024, All Rights Reserved #} +{% load origin static person_filters ietf_filters %} +{% block pagehead %} + +{% endblock %} +{% block title %}Group Leadership{% endblock %} +{% block content %} + {% origin %} +

Group Leadership ({{ group_type }})

+ {% if user|has_role:"Secretariat" %} +
+ + Export as CSV + +
+ {% endif %} + + + + + + + + + {% for leader in leaders %} + + + + + {% endfor %} + +
LeaderGroups
{{ leader.name }}{{ leader.groups }}
+{% endblock %} diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html index 8acc688cc..deaea1e67 100644 --- a/ietf/templates/group/meetings.html +++ b/ietf/templates/group/meetings.html @@ -40,7 +40,7 @@ {% if future %}

Future Meetings - @@ -174,4 +174,4 @@ ietf_timezone.initialize('local'); }); -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/meeting/group_proceedings.html b/ietf/templates/meeting/group_proceedings.html index 95d6dc5da..496fa9226 100644 --- a/ietf/templates/meeting/group_proceedings.html +++ b/ietf/templates/meeting/group_proceedings.html @@ -88,12 +88,13 @@
{% endfor %} - {% if entry.session.video_stream_url %} - - Session recording - -
- {% endif %} + {% for rec in entry.meetecho_recordings %} + + Session recording + {% if rec.time %}{{ rec.time|date:"D G:i"}}{% endif %} + +
+ {% endfor%} {# slides #} diff --git a/ietf/templates/meeting/important-dates.html b/ietf/templates/meeting/important-dates.html index 1d41b4a7f..568276f56 100644 --- a/ietf/templates/meeting/important-dates.html +++ b/ietf/templates/meeting/important-dates.html @@ -19,7 +19,7 @@ {% for meeting in meetings %} {% if meeting.show_important_dates %} -

+

IETF {{ meeting.number }}
{{ meeting.date }}, {{ meeting.city }}, {{ meeting.country }} diff --git a/ietf/templates/meeting/session_details_panel.html b/ietf/templates/meeting/session_details_panel.html index 52aeaaa8c..a0f5884b9 100644 --- a/ietf/templates/meeting/session_details_panel.html +++ b/ietf/templates/meeting/session_details_panel.html @@ -320,51 +320,50 @@ {% endif %} {# Recordings #} - {% if session.has_recordings %} - {% with session.recordings as recordings %} - {% if recordings %} - {# There's no guaranteed order, so this is a bit messy: #} - {# First, the audio recordings, if any #} - {% for r in recordings %} - {% if r.get_href and 'audio' in r.get_href %} - - - {{ r.title }} - - - {% endif %} - {% endfor %} - {# Then the youtube recordings #} - {% for r in recordings %} - {% if r.get_href and 'youtu' in r.get_href %} - - - {{ r.title }} - - - {% endif %} - {% endfor %} - {# Finally, any other recordings #} - {% for r in recordings %} - {% if r.get_href and not 'audio' in r.get_href and not 'youtu' in r.get_href %} - - - {{ r.title }} - - - {% endif %} - {% endfor %} - {% endif %} - {% endwith %} - {% if session.video_stream_url %} - - - - Session recording - - - + {% with session.recordings as recordings %} + {% if recordings %} + {# There's no guaranteed order, so this is a bit messy: #} + {# First, the audio recordings, if any #} + {% for r in recordings %} + {% if r.get_href and 'audio' in r.get_href %} + + + {{ r.title }} + + + {% endif %} + {% endfor %} + {# Then the youtube recordings #} + {% for r in recordings %} + {% if r.get_href and 'youtu' in r.get_href %} + + + {{ r.title }} + + + {% endif %} + {% endfor %} + {# Finally, any other recordings #} + {% for r in recordings %} + {% if r.get_href and not 'audio' in r.get_href and not 'youtu' in r.get_href %} + + + {{ r.title }} + + + {% endif %} + {% endfor %} {% endif %} + {% endwith %} + {% if session.session_recording_url %} + + + + + Meetecho session recording + + + {% endif %} diff --git a/ietf/utils/aliases.py b/ietf/utils/aliases.py deleted file mode 100644 index 9f9aebc0b..000000000 --- a/ietf/utils/aliases.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- -# -*- Python -*- -# -# $Id: aliasutil.py $ -# -# Author: Markus Stenberg -# - - -""" - -Mailing list alias dumping utilities - -""" - - -from django.conf import settings -from ietf.utils.log import log - -import debug # pyflakes:ignore - -def rewrite_email_address(email): - """ Prettify the email address (and if it's empty, skip it by - returning None). """ - if not email: - return - email = email.strip() - if not email: - return - if email[0]=='<' and email[-1] == '>': - email = email[1:-1] - # If it doesn't look like email, skip - if '@' not in email and '?' not in email: - return - return email - -def rewrite_address_list(l): - """ This utility function makes sure there is exactly one instance - of an address within the result list, and preserves order - (although it may not be relevant to start with) """ - h = {} - for address in l: - #address = address.strip() - if address in h: continue - h[address] = True - yield address - -def dump_sublist(afile, vfile, alias, adomains, vdomain, emails): - if not emails: - return emails - # Nones in the list should be skipped - emails = [_f for _f in emails if _f] - - # Make sure emails are sane and eliminate the Nones again for - # non-sane ones - emails = [rewrite_email_address(e) for e in emails] - emails = [_f for _f in emails if _f] - - # And we'll eliminate the duplicates too but preserve order - emails = list(rewrite_address_list(emails)) - if not emails: - return emails - try: - filtername = 'xfilter-%s' % (alias, ) # in aliases, --> | expandname - expandname = 'expand-%s' % (alias, ) # in virtual, --> email list - - for domain in adomains: - aliasaddr = '%s@%s' % (alias, domain) # in virtual, --> filtername - vfile.write('%-64s %s\n' % (aliasaddr, filtername)) - afile.write('%-64s "|%s filter %s %s"\n' % (filtername+':', settings.POSTCONFIRM_PATH, expandname, vdomain)) - vfile.write('%-64s %s\n' % ("%s@%s"%(expandname, vdomain), ', '.join(emails))) - - except UnicodeEncodeError: - # If there's unicode in email address, something is badly - # wrong and we just silently punt - # XXX - is there better approach? - log('Error encoding email address for an %s alias: %s' % (alias, repr(emails))) - return [] - return emails - diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 157f97912..215d78d65 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -12,6 +12,8 @@ try: from selenium import webdriver from selenium.webdriver.firefox.service import Service from selenium.webdriver.firefox.options import Options + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions from selenium.webdriver.common.by import By except ImportError as e: skip_selenium = True @@ -87,6 +89,48 @@ class IetfSeleniumTestCase(IetfLiveServerTestCase): # actions = ActionChains(self.driver) # actions.move_to_element(element).perform() + def scroll_and_click(self, element_locator, timeout_seconds=5): + """ + Selenium has restrictions around clicking elements outside the viewport, so + this wrapper encapsulates the boilerplate of forcing scrolling and clicking. + + :param element_locator: A two item tuple of a Selenium locator eg `(By.CSS_SELECTOR, '#something')` + """ + + # so that we can restore the state of the webpage after clicking + original_html_scroll_behaviour_to_restore = self.driver.execute_script('return document.documentElement.style.scrollBehavior') + original_html_overflow_to_restore = self.driver.execute_script('return document.documentElement.style.overflow') + + original_body_scroll_behaviour_to_restore = self.driver.execute_script('return document.body.style.scrollBehavior') + original_body_overflow_to_restore = self.driver.execute_script('return document.body.style.overflow') + + self.driver.execute_script('document.documentElement.style.scrollBehavior = "auto"') + self.driver.execute_script('document.documentElement.style.overflow = "auto"') + + self.driver.execute_script('document.body.style.scrollBehavior = "auto"') + self.driver.execute_script('document.body.style.overflow = "auto"') + + element = self.driver.find_element(element_locator[0], element_locator[1]) + self.scroll_to_element(element) + + # Note that Selenium itself seems to have multiple definitions of 'clickable'. + # You might expect that the following wait for the 'element_to_be_clickable' + # would confirm that the following .click() would succeed but it doesn't. + # That's why the preceeding code attempts to force scrolling to bring the + # element into the viewport to allow clicking. + WebDriverWait(self.driver, timeout_seconds).until(expected_conditions.element_to_be_clickable(element_locator)) + + element.click() + + if original_html_scroll_behaviour_to_restore: + self.driver.execute_script(f'document.documentElement.style.scrollBehavior = "{original_html_scroll_behaviour_to_restore}"') + if original_html_overflow_to_restore: + self.driver.execute_script(f'document.documentElement.style.overflow = "{original_html_overflow_to_restore}"') + + if original_body_scroll_behaviour_to_restore: + self.driver.execute_script(f'document.body.style.scrollBehavior = "{original_body_scroll_behaviour_to_restore}"') + if original_body_overflow_to_restore: + self.driver.execute_script(f'document.body.style.overflow = "{original_body_overflow_to_restore}"') class presence_of_element_child_by_css_selector: """Wait for presence of a child of a WebElement matching a CSS selector diff --git a/k8s/README.md b/k8s/README.md index 73b597867..3966101ab 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -1,5 +1,5 @@ -# Kustomize deployment - -## Run locally - +# Kustomize deployment + +## Run locally + The `secrets.yaml` file is provided as a reference only and must be referenced manually in the `kustomization.yaml` file. \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 4b79f0075..2b623da2b 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -1,16 +1,16 @@ -namespace: datatracker -namePrefix: dt- -configMapGenerator: - - name: files-cfgmap - files: - - nginx-logging.conf - - nginx-auth.conf - - nginx-datatracker.conf - - settings_local.py -resources: - - auth.yaml - - beat.yaml - - celery.yaml - - datatracker.yaml - - memcached.yaml - - rabbitmq.yaml +namespace: datatracker +namePrefix: dt- +configMapGenerator: + - name: files-cfgmap + files: + - nginx-logging.conf + - nginx-auth.conf + - nginx-datatracker.conf + - settings_local.py +resources: + - auth.yaml + - beat.yaml + - celery.yaml + - datatracker.yaml + - memcached.yaml + - rabbitmq.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 4b362c88c..5a4c9f0ae 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -1,80 +1,80 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: memcached -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: memcached - template: - metadata: - labels: - app: memcached - spec: - securityContext: - runAsNonRoot: true - containers: - # ----------------------------------------------------- - # Memcached - # ----------------------------------------------------- - - image: "memcached:1.6-alpine" - imagePullPolicy: IfNotPresent - args: ["-m", "1024"] - name: memcached - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # memcached image sets up uid/gid 11211 - runAsUser: 11211 - runAsGroup: 11211 - # ----------------------------------------------------- - # Memcached Exporter for Prometheus - # ----------------------------------------------------- - - image: "quay.io/prometheus/memcached-exporter:v0.14.3" - imagePullPolicy: IfNotPresent - name: memcached-exporter - ports: - - name: metrics - containerPort: 9150 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 65534 # nobody - runAsGroup: 65534 # nobody - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: memcached - annotations: - k8s.grafana.com/scrape: "true" # this is not a bool - k8s.grafana.com/metrics.portName: "metrics" -spec: - type: ClusterIP - ports: - - port: 11211 - targetPort: memcached - protocol: TCP - name: memcached - - port: 9150 - targetPort: metrics - protocol: TCP - name: metrics - selector: - app: memcached +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # Memcached + # ----------------------------------------------------- + - image: "memcached:1.6-alpine" + imagePullPolicy: IfNotPresent + args: ["-m", "1024"] + name: memcached + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 + # ----------------------------------------------------- + # Memcached Exporter for Prometheus + # ----------------------------------------------------- + - image: "quay.io/prometheus/memcached-exporter:v0.14.3" + imagePullPolicy: IfNotPresent + name: memcached-exporter + ports: + - name: metrics + containerPort: 9150 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 # nobody + runAsGroup: 65534 # nobody + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached + annotations: + k8s.grafana.com/scrape: "true" # this is not a bool + k8s.grafana.com/metrics.portName: "metrics" +spec: + type: ClusterIP + ports: + - port: 11211 + targetPort: memcached + protocol: TCP + name: memcached + - port: 9150 + targetPort: metrics + protocol: TCP + name: metrics + selector: + app: memcached diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml index 4e76a86a5..ba90af9c2 100644 --- a/k8s/secrets.yaml +++ b/k8s/secrets.yaml @@ -1,83 +1,83 @@ -apiVersion: v1 -kind: Secret -metadata: - name: secrets-env -type: Opaque -stringData: - DATATRACKER_SERVER_MODE: "development" # development for staging, production for production - DATATRACKER_ADMINS: |- - Robert Sparks - Ryan Cross - Kesara Rathnayake - Jennifer Richards - Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed - # DATATRACKER_DATATRACKER_DEBUG: "false" - - # DB access details - needs to be filled in - # DATATRACKER_DB_HOST: "db" - # DATATRACKER_DB_PORT: "5432" - # DATATRACKER_DB_NAME: "datatracker" - # DATATRACKER_DB_USER: "django" # secret - # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret - # DATATRACKER_DB_CONN_MAX_AGE: "0" # connection per request if not set, no limit if set to "None" - # DATATRACKER_DB_CONN_HEALTH_CHECKS: "false" - - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - - # Set this to point testing / staging at the production statics server until we - # sort that out - # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - - # DATATRACKER_EMAIL_DEBUG: "true" - - # Outgoing email details - # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost - # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 - - # The value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret - - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret - - # API key configuration - DATATRACKER_API_KEY_TYPE: "ES265" - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS - 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel - ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM - weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O - UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 - F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY - yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 - N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= - - #DATATRACKER_REGISTRATION_API_KEY: "some-key" # secret" - - # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" - DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret - DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret - - # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo - # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" - - CELERY_PASSWORD: "this-is-a-secret" # secret - - # Only one of these may be set - # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret - # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret - - # use this to override default - one entry per line - # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- - # https://datatracker.staging.ietf.org - - # Scout configuration - DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" +apiVersion: v1 +kind: Secret +metadata: + name: secrets-env +type: Opaque +stringData: + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DB_HOST: "db" + # DATATRACKER_DB_PORT: "5432" + # DATATRACKER_DB_NAME: "datatracker" + # DATATRACKER_DB_USER: "django" # secret + # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret + # DATATRACKER_DB_CONN_MAX_AGE: "0" # connection per request if not set, no limit if set to "None" + # DATATRACKER_DB_CONN_HEALTH_CHECKS: "false" + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + #DATATRACKER_REGISTRATION_API_KEY: "some-key" # secret" + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + # Only one of these may be set + # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret + # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file