From 7905b8db611dacb1389a13f33146fd019c723907 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Thu, 14 Nov 2024 01:53:55 -0500 Subject: [PATCH] ci: Update build.yml to use GHA larger runner --- .github/workflows/build.yml | 293 ++++++++++++------------------------ 1 file changed, 93 insertions(+), 200 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 405a79326..67f24ba76 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -149,7 +149,8 @@ jobs: name: Make Release if: ${{ !failure() && !cancelled() }} needs: [tests, prepare] - runs-on: ubuntu-latest + runs-on: + group: hperf-8c32r permissions: contents: write packages: write @@ -166,215 +167,107 @@ jobs: fetch-depth: 1 fetch-tags: false - - name: Launch build VM - id: azlaunch - timeout-minutes: 10 - run: | - echo "Authenticating to Azure..." - az login --service-principal -u ${{ secrets.AZ_BUILD_APP_ID }} -p ${{ secrets.AZ_BUILD_PWD }} --tenant ${{ secrets.AZ_BUILD_TENANT_ID }} - - echo "Creating VM..." - vminfo=$(az vm create \ - --resource-group ghaDatatracker \ - --name tmpGhaBuildVM-${{ github.run_number }} \ - --image Ubuntu2204 \ - --admin-username azureuser \ - --generate-ssh-keys \ - --priority Spot \ - --size Standard_D8ads_v5 \ - --max-price -1 \ - --ephemeral-os-disk \ - --os-disk-size-gb 100 \ - --eviction-policy Delete \ - --nic-delete-option Delete \ - --os-disk-delete-option Delete \ - --output tsv \ - --query "publicIpAddress") - echo "ipaddr=$vminfo" >> "$GITHUB_OUTPUT" - echo "VM Public IP: $vminfo" - cat ~/.ssh/id_rsa > ${{ github.workspace }}/prvkey.key - - echo "Fetching SSH host public keys..." - until ssh-keyscan -t rsa $vminfo 2> /dev/null - do - echo "Will try again in 5 seconds..." - sleep 5 - done - ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: 18.x - - name: Remote SSH into Build VM - uses: appleboy/ssh-action@25ce8cbbcb08177468c7ff7ec5cbfa236f9341e1 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Download a Coverage Results + if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} + uses: actions/download-artifact@v4.1.8 + with: + name: coverage + + - name: Make Release Build env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_ACTOR: ${{ github.actor }} - GITHUB_SHA: ${{ github.sha }} - GITHUB_REF_NAME: ${{ github.ref_name }} - GITHUB_RUN_ID: ${{ github.run_id }} + DEBIAN_FRONTEND: noninteractive + BROWSERSLIST_IGNORE_OLD_DATA: 1 + run: | + echo "PKG_VERSION: $PKG_VERSION" + echo "GITHUB_SHA: $GITHUB_SHA" + echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" + echo "Running frontend build script..." + echo "Compiling native node packages..." + yarn rebuild + echo "Packaging static assets..." + yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ + yarn legacy:build + echo "Setting version $PKG_VERSION..." + sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py + sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py + sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py + + - name: Set Production Flags + if: ${{ env.SHOULD_DEPLOY == 'true' }} + run: | + echo "Setting production flags in settings.py..." + sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py + + - name: Make Release Tarball + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Build release tarball..." + mkdir -p /home/runner/work/release + tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt . + + - name: Collect + Push Statics + env: + DEBIAN_FRONTEND: noninteractive AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }} AWS_DEFAULT_REGION: auto AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }} - PKG_VERSION: ${{ env.PKG_VERSION }} - SHOULD_DEPLOY: ${{ env.SHOULD_DEPLOY }} - SKIP_TESTS: ${{ github.event.inputs.skiptests }} + run: | + echo "Collecting statics..." + echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" + docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh + echo "Pushing statics..." + cd static + aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors + + - name: Augment dockerignore for docker image build + env: DEBIAN_FRONTEND: noninteractive - BROWSERSLIST_IGNORE_OLD_DATA: 1 - TARGET_BASE: ${{ env.TARGET_BASE }} + run: | + cat >> .dockerignore < /dev/null \ - && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ - && sudo apt update \ - && sudo apt install gh -y - - echo "==========================================================================" - echo "Installing AWS CLI..." - echo "==========================================================================" - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - sudo ./aws/install - - echo "==========================================================================" - echo "Install Node.js..." - echo "==========================================================================" - curl -fsSL https://deb.nodesource.com/setup_18.x -o nodesource_setup.sh - sudo bash nodesource_setup.sh - sudo apt-get install -y nodejs - sudo corepack enable - - echo "==========================================================================" - echo "Install Python 3.x..." - echo "==========================================================================" - sudo apt-get install python3 python3-dev -y - python3 --version - - echo "==========================================================================" - echo "Clone project..." - echo "==========================================================================" - sudo mkdir -p /workspace - sudo chown azureuser /workspace - cd /workspace - gh repo clone ietf-tools/datatracker -- --depth=1 --no-tags --branch ${{ github.ref_name }} - cd datatracker - - if [ "$SKIP_TESTS" = "false" ] || [ "$GITHUB_REF_NAME" = "release" ] ; then - echo "==========================================================================" - echo "Downloading coverage..." - echo "==========================================================================" - gh run download $GITHUB_RUN_ID -n coverage - fi - - echo "==========================================================================" - echo "Building project..." - echo "==========================================================================" - echo "PKG_VERSION: $PKG_VERSION" - echo "GITHUB_SHA: $GITHUB_SHA" - echo "GITHUB_REF_NAME: $GITHUB_REF_NAME" - echo "Running frontend build script..." - echo "Compiling native node packages..." - yarn rebuild - echo "Packaging static assets..." - yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/ - yarn legacy:build - echo "Setting version $PKG_VERSION..." - sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py - sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py - sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py - - if [ "$SHOULD_DEPLOY" = "true" ] ; then - echo "==========================================================================" - echo "Setting production flags in settings.py..." - echo "==========================================================================" - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py - fi - - echo "==========================================================================" - echo "Build release tarball..." - echo "==========================================================================" - mkdir -p /workspace/release - tar -czf /workspace/release.tar.gz -X dev/build/exclude-patterns.txt . - - echo "==========================================================================" - echo "Collecting statics..." - echo "==========================================================================" - echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}" - sudo docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh - echo "Pushing statics..." - cd static - aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors - cd .. - - echo "==========================================================================" - echo "Augment dockerignore for docker image build..." - echo "==========================================================================" - cat >> .dockerignore <