diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5884b724f..31150ed99 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,12 +45,19 @@ jobs: echo "Running tests..." if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then echo "Lower coverage failures will be ignored." - ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage else - ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test fi coverage xml + - name: Upload geckodriver.log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: geckodriverlog + path: geckodriver.log + - name: Upload Coverage Results to Codecov uses: codecov/codecov-action@v3.1.4 with: @@ -161,4 +168,4 @@ jobs: with: name: playwright-legacy-results-${{ matrix.project }} path: playwright/test-results/ - if-no-files-found: ignore \ No newline at end of file + if-no-files-found: ignore diff --git a/dev/tests/debug.sh b/dev/tests/debug.sh index 405daae37..8ed28371a 100644 --- a/dev/tests/debug.sh +++ b/dev/tests/debug.sh @@ -12,7 +12,7 @@ echo "Fetching latest images..." docker pull ghcr.io/ietf-tools/datatracker-app-base:latest docker pull ghcr.io/ietf-tools/datatracker-db:latest echo "Starting containers..." -docker compose -f docker-compose.debug.yml -p dtdebug up -d +docker compose -f docker-compose.debug.yml -p dtdebug --compatibility up -d echo "Copying working directory into container..." docker compose -p dtdebug cp ../../. app:/__w/datatracker/datatracker/ echo "Run prepare script..." diff --git a/dev/tests/docker-compose.debug.yml b/dev/tests/docker-compose.debug.yml index 74491a5b2..6362ef072 100644 --- a/dev/tests/docker-compose.debug.yml +++ b/dev/tests/docker-compose.debug.yml @@ -16,6 +16,12 @@ services: CI: 'true' GITHUB_ACTIONS: 'true' HOME: /github/home + deploy: + resources: + limits: + cpus: '2' + memory: '7GB' + db: image: ghcr.io/ietf-tools/datatracker-db:latest restart: unless-stopped diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 5401007fd..12d03d74d 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -19,7 +19,7 @@ RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesourc RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - + # Add PostgreSQL Source RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - @@ -83,15 +83,6 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends # Install kramdown-rfc2629 (ruby) RUN gem install kramdown-rfc2629 -# Install chromedriver -COPY docker/scripts/app-install-chromedriver.sh /tmp/app-install-chromedriver.sh -RUN sed -i 's/\r$//' /tmp/app-install-chromedriver.sh && \ - chmod +x /tmp/app-install-chromedriver.sh -RUN /tmp/app-install-chromedriver.sh - -# Fix /dev/shm permissions for chromedriver -RUN chmod 1777 /dev/shm - # GeckoDriver ARG GECKODRIVER_VERSION=latest RUN GK_VERSION=$(if [ ${GECKODRIVER_VERSION:-latest} = "latest" ]; then echo "0.33.0"; else echo $GECKODRIVER_VERSION; fi) \ diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 7e58e797c..c8286b242 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -18,9 +18,6 @@ sudo chown -R dev:dev "$WORKSPACEDIR/.vite" sudo chown -R dev:dev "$WORKSPACEDIR/.yarn/unplugged" sudo chown dev:dev "/assets" -echo "Fix chromedriver /dev/shm permissions..." -sudo chmod 1777 /dev/shm - # Run nginx echo "Starting nginx..." sudo nginx diff --git a/docker/scripts/app-install-chromedriver.sh b/docker/scripts/app-install-chromedriver.sh deleted file mode 100755 index 43532a1cf..000000000 --- a/docker/scripts/app-install-chromedriver.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -HOSTARCH=$(arch) -if [ $HOSTARCH == "x86_64" ]; then - echo "Installing chrome driver..." - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - - echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list - apt-get update -y - apt-get install -y google-chrome-stable - CHROMEVER=$(google-chrome --product-version | grep -o "[^\.]*\.[^\.]*\.[^\.]*") - DRIVERVER=$(curl -s "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_$CHROMEVER") - wget -q --continue -P /chromedriver "http://chromedriver.storage.googleapis.com/$DRIVERVER/chromedriver_linux64.zip" - unzip /chromedriver/chromedriver* -d /chromedriver - ln -s /chromedriver/chromedriver /usr/local/bin/chromedriver - ln -s /chromedriver/chromedriver /usr/bin/chromedriver -else - echo "This architecture doesn't support chromedriver. Skipping installation..." -fi \ No newline at end of file diff --git a/ietf/doc/tests_js.py b/ietf/doc/tests_js.py index ac63c0995..acd74c4a0 100644 --- a/ietf/doc/tests_js.py +++ b/ietf/doc/tests_js.py @@ -41,7 +41,7 @@ class EditAuthorsTests(IetfSeleniumTestCase): (By.CSS_SELECTOR, result_selector), name )) - input.send_keys('\n') # select the object + self.driver.find_element(By.CSS_SELECTOR, result_selector).click() # After the author is selected, the email select options will be populated. # Wait for that, then click on the option corresponding to the requested email. @@ -94,10 +94,8 @@ class EditAuthorsTests(IetfSeleniumTestCase): # get the "add author" button so we can add blank author forms add_author_button = self.driver.find_element(By.ID, 'add-author-button') for index, auth in enumerate(authors): - self.driver.execute_script("arguments[0].scrollIntoView();", add_author_button) # FIXME: no idea why this fails: - # self.scroll_to_element(add_author_button) # Can only click if it's in view! - self.driver.execute_script("arguments[0].click();", add_author_button) # FIXME: no idea why this fails: - # add_author_button.click() # Create a new form. Automatically scrolls to it. + self.scroll_to_element(add_author_button) # Can only click if it's in view! + add_author_button.click() # Create a new form. Automatically scrolls to it. author_forms = authors_list.find_elements(By.CLASS_NAME, 'author-panel') authors_added = index + 1 self.assertEqual(len(author_forms), authors_added + 1) # Started with 1 author, hence +1 @@ -119,9 +117,8 @@ class EditAuthorsTests(IetfSeleniumTestCase): self.driver.find_element(By.ID, 'id_basis').send_keys('change testing') # Now click the 'submit' button and check that the update was accepted. submit_button = self.driver.find_element(By.CSS_SELECTOR, '#content button[type="submit"]') - self.driver.execute_script("arguments[0].click();", submit_button) # FIXME: no idea why this fails: - # self.scroll_to_element(submit_button) - # submit_button.click() + self.scroll_to_element(submit_button) + submit_button.click() # Wait for redirect to the document_main view self.wait.until( expected_conditions.url_to_be( diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index e69afe5ca..e299cfef6 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -499,7 +499,7 @@ class EditMeetingScheduleTests(IetfSeleniumTestCase): clicked_index = 1 # scroll so the button we want to click is just below the navbar, otherwise it may # fall beneath the sessions panel - navbar = self.driver.find_element_by_class_name('navbar') + navbar = self.driver.find_element(By.CSS_SELECTOR, '.navbar') self.driver.execute_script( 'window.scrollBy({top: %s, behavior: "instant"})' % ( future_swap_days_buttons[1].location['y'] - navbar.size['height'] @@ -1232,10 +1232,13 @@ class InterimTests(IetfSeleniumTestCase): self.driver.get(self.absreverse('ietf.meeting.views.upcoming')) tz_select_input = self.driver.find_element(By.ID, 'timezone-select') tz_select_bottom_input = self.driver.find_element(By.ID, 'timezone-select-bottom') - local_tz_link = self.driver.find_element(By.ID, 'local-timezone') - utc_tz_link = self.driver.find_element(By.ID, 'utc-timezone') - local_tz_bottom_link = self.driver.find_element(By.ID, 'local-timezone-bottom') - utc_tz_bottom_link = self.driver.find_element(By.ID, 'utc-timezone-bottom') + + # For things we click, need to click the labels / actually visible items. The actual inputs are hidden + # and managed by the JS. + local_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone"]') + utc_tz_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone"]') + local_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="local-timezone-bottom"]') + utc_tz_bottom_link = self.driver.find_element(By.CSS_SELECTOR, 'label[for="utc-timezone-bottom"]') # wait for the select box to be updated - look for an arbitrary time zone to be in # its options list to detect this @@ -1245,7 +1248,10 @@ class InterimTests(IetfSeleniumTestCase): (By.CSS_SELECTOR, '#timezone-select > option[value="%s"]' % arbitrary_tz) ) ) - + tz_selector_clickables = self.driver.find_elements(By.CSS_SELECTOR, ".tz-display .select2") + self.assertEqual(len(tz_selector_clickables), 2) + (tz_selector_top, tz_selector_bottom) = tz_selector_clickables + arbitrary_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, '#timezone-select-bottom > option[value="%s"]' % arbitrary_tz) @@ -1266,8 +1272,7 @@ class InterimTests(IetfSeleniumTestCase): _assert_ietf_tz_correct(ietf_meetings, local_tz) # click 'utc' button - self.driver.execute_script("arguments[0].click();", utc_tz_link) # FIXME-LARS: not working: - # utc_tz_link.click() + utc_tz_link.click() self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1279,8 +1284,7 @@ class InterimTests(IetfSeleniumTestCase): _assert_ietf_tz_correct(ietf_meetings, 'UTC') # click back to 'local' - self.driver.execute_script("arguments[0].click();", local_tz_link) # FIXME-LARS: not working: - # local_tz_link.click() + local_tz_link.click() self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt)) self.assertTrue(local_tz_opt.is_selected()) self.assertTrue(local_tz_bottom_opt.is_selected()) @@ -1292,7 +1296,12 @@ class InterimTests(IetfSeleniumTestCase): _assert_ietf_tz_correct(ietf_meetings, local_tz) # Now select a different item from the select input - arbitrary_tz_opt.click() + tz_selector_top.click() + self.wait.until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]') + ) + ).click() self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1305,8 +1314,8 @@ class InterimTests(IetfSeleniumTestCase): # Now repeat those tests using the widgets at the bottom of the page # click 'utc' button - self.driver.execute_script("arguments[0].click();", utc_tz_bottom_link) # FIXME-LARS: not working: - # utc_tz_bottom_link.click() + self.scroll_to_element(utc_tz_bottom_link) + utc_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(utc_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1318,8 +1327,8 @@ class InterimTests(IetfSeleniumTestCase): _assert_ietf_tz_correct(ietf_meetings, 'UTC') # click back to 'local' - self.driver.execute_script("arguments[0].click();", local_tz_bottom_link) # FIXME-LARS: not working: - # local_tz_bottom_link.click() + self.scroll_to_element(local_tz_bottom_link) + local_tz_bottom_link.click() self.wait.until(expected_conditions.element_to_be_selected(local_tz_opt)) self.assertTrue(local_tz_opt.is_selected()) self.assertTrue(local_tz_bottom_opt.is_selected()) @@ -1331,7 +1340,13 @@ class InterimTests(IetfSeleniumTestCase): _assert_ietf_tz_correct(ietf_meetings, local_tz) # Now select a different item from the select input - arbitrary_tz_bottom_opt.click() + self.scroll_to_element(tz_selector_bottom) + tz_selector_bottom.click() + self.wait.until( + expected_conditions.presence_of_element_located( + (By.CSS_SELECTOR, 'span.select2-container .select2-results li[id$="America/Halifax"]') + ) + ).click() self.wait.until(expected_conditions.element_to_be_selected(arbitrary_tz_opt)) self.assertFalse(local_tz_opt.is_selected()) self.assertFalse(local_tz_bottom_opt.is_selected()) @@ -1382,6 +1397,7 @@ class InterimTests(IetfSeleniumTestCase): ), 'Modal close button not found or not clickable', ) + time.sleep(0.3) # gross, but the button is clickable while still fading in close_modal_button.click() self.wait.until( expected_conditions.invisibility_of_element(modal_div), diff --git a/ietf/settings.py b/ietf/settings.py index 34076f329..d0cb6b852 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -572,8 +572,6 @@ GLOBAL_TEST_FIXTURES = [ 'names','ietf.utils.test_data.make_immutable_base_data' TEST_DIFF_FAILURE_DIR = "/tmp/test/failure/" -TEST_GHOSTDRIVER_LOG_PATH = "ghostdriver.log" - # These are regexes TEST_URL_COVERAGE_EXCLUDE = [ r"^\^admin/", diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index a901df66f..07d6ed9dd 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2014-2021, All Rights Reserved # -*- coding: utf-8 -*- -from django.conf import settings +import os + from django.urls import reverse as urlreverse from unittest import skipIf @@ -9,10 +10,9 @@ skip_selenium = False skip_message = "" try: from selenium import webdriver - from selenium.webdriver.chrome.service import Service - from selenium.webdriver.chrome.options import Options + from selenium.webdriver.firefox.service import Service + from selenium.webdriver.firefox.options import Options from selenium.webdriver.common.by import By - from selenium.webdriver.common.desired_capabilities import DesiredCapabilities except ImportError as e: skip_selenium = True skip_message = "Skipping selenium tests: %s" % e @@ -21,7 +21,7 @@ except ImportError as e: from ietf.utils.pipe import pipe from ietf.utils.test_runner import IetfLiveServerTestCase -executable_name = 'chromedriver' +executable_name = 'geckodriver' code, out, err = pipe('{} --version'.format(executable_name)) if code != 0: skip_selenium = True @@ -30,20 +30,11 @@ if skip_selenium: print(" "+skip_message) def start_web_driver(): - service = Service(executable_path="chromedriver", - log_path=settings.TEST_GHOSTDRIVER_LOG_PATH) - service.start() + service = Service(log_output=f"{executable_name}.log", service_args=['--log-no-truncate']) options = Options() - options.add_argument("headless") - options.add_argument("disable-extensions") - options.add_argument("disable-gpu") # headless needs this - options.add_argument("no-sandbox") # docker needs this - dc = DesiredCapabilities.CHROME - dc["goog:loggingPrefs"] = {"browser": "ALL"} - # For selenium 3: - return webdriver.Chrome("chromedriver", options=options, desired_capabilities=dc) - # For selenium 4: - # return webdriver.Chrome(service=service, options=options, desired_capabilities=dc) + options.add_argument("--headless") + os.environ["MOZ_REMOTE_SETTINGS_DEVTOOLS"] = "1" + return webdriver.Firefox(service=service, options=options) def selenium_enabled(): diff --git a/requirements.txt b/requirements.txt index 33ade01c8..e627d0767 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,12 +65,12 @@ types-requests>=2.27.1 requests-mock>=1.9.3 rfc2html>=2.0.3 scout-apm>=2.24.2 -selenium>=3.141.0,<4.0 +selenium>=4.0 tblib>=1.7.0 # So that the django test runner provides tracebacks tlds>=2022042700 # Used to teach bleach about which TLDs currently exist tqdm>=4.64.0 Unidecode>=1.3.4 -urllib3<2 # v2 causes selenium tests to fail with "Timeout value was =2 weasyprint>=59 xml2rfc>=3.12.4 xym>=0.6,<1.0