datatracker/.github/workflows/build.yml

name: Build and Release

on:
  push:
    tags:
      - '*'

  workflow_dispatch:
    inputs:
      summary:
        description: 'Release Summary'
        required: false
        type: string
        default: ''
      publish:
        description: 'Create Production Release'
        default: false
        required: true
        type: boolean
      sandbox:
        description: 'Deploy to Sandbox'
        default: true
        required: true
        type: boolean
      skiptests:
        description: 'Skip Tests'
        default: false
        required: true
        type: boolean
      ignoreLowerCoverage:
        description: 'Ignore Lower Coverage'
        default: false
        required: true
        type: boolean
      updateCoverage:
        description: 'Update Baseline Coverage'
        default: false
        required: true
        type: boolean
      dryrun:
        description: 'Dry Run'
        default: false
        required: true
        type: boolean

jobs:
  # -----------------------------------------------------------------
  # PREPARE
  # -----------------------------------------------------------------
  prepare:
    name: Prepare Release
    runs-on: ubuntu-latest
    outputs:
      should_deploy: ${{ steps.buildvars.outputs.should_deploy }}
      pkg_version: ${{ steps.buildvars.outputs.pkg_version }}
      from_tag: ${{ steps.semver.outputs.nextStrict }}
      to_tag: ${{ steps.semver.outputs.current }}

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        
    - name: Dry Run Notify
      if: ${{ github.event.inputs.dryrun == 'true' }}
      run: |
        echo "::notice::This is a DRY RUN of a production release. No release will be created."        

    - name: Get Next Version
      if: ${{ github.event.inputs.publish == 'true' || github.event.inputs.dryrun == 'true' }}
      id: semver
      uses: ietf-tools/semver-action@v1
      with:
        token: ${{ github.token }}
        branch: main
        skipInvalidTags: true
        
    - name: Set Next Version Env Var
      if: ${{ github.event.inputs.publish == 'true' || github.event.inputs.dryrun == 'true' }}
      run: |
        echo "NEXT_VERSION=$nextStrict" >> $GITHUB_ENV
                
    - name: Create Draft Release
      uses: ncipollo/release-action@v1.11.1
      if: ${{ github.event.inputs.publish == 'true' && github.event.inputs.dryrun == 'false' }}
      with:
        prerelease: true
        draft: false
        commit: ${{ github.sha }}
        tag: ${{ env.NEXT_VERSION }}
        name: ${{ env.NEXT_VERSION }}
        body: '*pending*'
        token: ${{ secrets.GITHUB_TOKEN }}
    
    - name: Set Build Variables
      id: buildvars
      run: |
        if [[ $NEXT_VERSION ]]; then
          echo "Using AUTO SEMVER mode: $NEXT_VERSION"
          echo "should_deploy=true" >> $GITHUB_OUTPUT
          echo "pkg_version=$NEXT_VERSION" >> $GITHUB_OUTPUT
          echo "::notice::Release $NEXT_VERSION created using branch $GITHUB_REF_NAME"
        elif [[ "$GITHUB_REF" =~ ^refs/tags/* ]]; then
          echo "Using TAG mode: $GITHUB_REF_NAME"
          echo "should_deploy=true" >> $GITHUB_OUTPUT
          echo "pkg_version=$GITHUB_REF_NAME" >> $GITHUB_OUTPUT
          echo "::notice::Release $GITHUB_REF_NAME created using tag $GITHUB_REF_NAME"
        else
          echo "Using TEST mode: 9.0.0-dev.$GITHUB_RUN_NUMBER"
          echo "should_deploy=false" >> $GITHUB_OUTPUT
          echo "pkg_version=9.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT
          echo "::notice::Non-production build 9.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
        fi        

  # -----------------------------------------------------------------
  # TESTS
  # -----------------------------------------------------------------
  tests-python:
    name: Run Tests (Python)
    if: ${{ github.event.inputs.skiptests == 'false' }}
    needs: [prepare]
    runs-on: ubuntu-latest
    container: ghcr.io/ietf-tools/datatracker-app-base:latest
    
    services:
      db:
        image: ghcr.io/ietf-tools/datatracker-db:latest
        volumes:
            - mariadb-data:/var/lib/mysql
        env:
            MYSQL_ROOT_PASSWORD: ietf
            MYSQL_DATABASE: ietf_utf8
            MYSQL_USER: django
            MYSQL_PASSWORD: RkTkDPFnKpko
    
    steps:
    - uses: actions/checkout@v3
    
    - name: Prepare for tests
      run: |
        chmod +x ./dev/tests/prepare.sh
        sh ./dev/tests/prepare.sh
                
    - name: Ensure DB is ready
      run: |
        /usr/local/bin/wait-for db:3306 -- echo "DB ready"
                
    - name: Run all tests
      shell: bash
      run: |
        echo "Running checks..."
        ./ietf/manage.py check
        ./ietf/manage.py migrate
        echo "Validating migrations..."
        if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then
          echo "Model changes without migrations found."
          exit 1
        fi
        echo "Running tests..."
        if [[ "x${{ github.event.inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then
          echo "Lower coverage failures will be ignored."
          ./ietf/manage.py test --settings=settings_sqlitetest --ignore-lower-coverage
        else
          ./ietf/manage.py test --settings=settings_sqlitetest
        fi
        coverage xml        

    - name: Upload Coverage Results to Codecov
      uses: codecov/codecov-action@v2.1.0
      with:
        files: coverage.xml
        
    - name: Convert Coverage Results
      if: ${{ always() }}
      run: |
        mv latest-coverage.json coverage.json
                
    - name: Upload Coverage Results as Build Artifact
      uses: actions/upload-artifact@v3
      if: ${{ always() }}
      with:
        name: coverage
        path: coverage.json

  tests-playwright:
    name: Run Tests (Playwright)
    if: ${{ github.event.inputs.skiptests == 'false' }}
    needs: [prepare]
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        project: [chromium, firefox]
    
    steps:
    - uses: actions/checkout@v3

    - uses: actions/setup-node@v3
      with:
        node-version: '18'

    - name: Run all tests
      run: |
        echo "Installing dependencies..."
        yarn
        echo "Installing Playwright..."
        cd playwright
        mkdir test-results
        npm ci
        npx playwright install --with-deps ${{ matrix.project }}
        echo "Running tests..."
        npx playwright test --project=${{ matrix.project }}
                
    - name: Upload Report
      uses: actions/upload-artifact@v3
      if: ${{ always() }}
      continue-on-error: true
      with:
        name: playwright-results-${{ matrix.project }}
        path: playwright/test-results/
        if-no-files-found: ignore

  tests-playwright-legacy:
    name: Run Tests (Playwright Legacy)
    if: ${{ github.event.inputs.skiptests == 'false' }}
    needs: [prepare]
    runs-on: ubuntu-latest
    container: ghcr.io/ietf-tools/datatracker-app-base:latest
    
    services:
      db:
        image: ghcr.io/ietf-tools/datatracker-db:latest
        volumes:
            - mariadb-data:/var/lib/mysql
        env:
            MYSQL_ROOT_PASSWORD: ietf
            MYSQL_DATABASE: ietf_utf8
            MYSQL_USER: django
            MYSQL_PASSWORD: RkTkDPFnKpko
    
    steps:
    - uses: actions/checkout@v3
    
    - name: Prepare for tests
      run: |
        chmod +x ./dev/tests/prepare.sh
        sh ./dev/tests/prepare.sh
                
    - name: Ensure DB is ready
      run: |
        /usr/local/bin/wait-for db:3306 -- echo "DB ready"
                
    - name: Start Datatracker
      run: |
        echo "Running checks..."
        ./ietf/manage.py check
        echo "Starting datatracker..."
        ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local &
        echo "Waiting for datatracker to be ready..."
        /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready"        

    - name: Run all tests
      env:
        # Required to get firefox to run as root:
        HOME: ""
      run: |
        echo "Installing dependencies..."
        yarn
        echo "Installing Playwright..."
        cd playwright
        mkdir test-results
        npm ci
        npx playwright install --with-deps ${{ matrix.project }}
        echo "Running tests..."
        npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js
                
    - name: Upload Report
      uses: actions/upload-artifact@v3
      if: ${{ always() }}
      continue-on-error: true
      with:
        name: playwright-legacy-results-${{ matrix.project }}
        path: playwright/test-results/
        if-no-files-found: ignore
        
  # -----------------------------------------------------------------
  # RELEASE
  # -----------------------------------------------------------------
  release:
    name: Make Release
    if: ${{ always() }}
    needs: [tests-python, tests-playwright, tests-playwright-legacy, prepare]
    runs-on: ubuntu-latest
    env:
      SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}}
      PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
      FROM_TAG: ${{needs.prepare.outputs.from_tag}}
      TO_TAG: ${{needs.prepare.outputs.to_tag}}

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        
    - name: Setup Node.js
      uses: actions/setup-node@v3
      with:
        node-version: '16'
    
    - name: Setup Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
        
    - name: Download a Coverage Results
      if: ${{ github.event.inputs.skiptests == 'false' }}
      uses: actions/download-artifact@v3.0.0
      with:
        name: coverage

    - name: Make Release Build
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        echo "PKG_VERSION: $PKG_VERSION"
        echo "GITHUB_SHA: $GITHUB_SHA"
        echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
        echo "Running build script..."
        chmod +x ./dev/deploy/build.sh
        sh ./dev/deploy/build.sh
        echo "Setting version $PKG_VERSION..."
        sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
        sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
        sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py        

    - name: Set Production Flags
      if: ${{ env.SHOULD_DEPLOY == 'true' }}
      run: |
        echo "Setting production flags in settings.py..."
        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
                
    - name: Make Release Tarball
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        echo "Build release tarball..."
        mkdir -p /home/runner/work/release
        tar -czf /home/runner/work/release/release.tar.gz -X dev/deploy/exclude-patterns.txt .
                
    - name: Update CHANGELOG
      id: changelog
      uses: Requarks/changelog-action@v1
      if: ${{ env.SHOULD_DEPLOY == 'true' && github.event.inputs.dryrun == 'false' }}
      with:
        token: ${{ github.token }}
        fromTag: ${{ env.FROM_TAG }}
        toTag: ${{ env.TO_TAG }}
        writeToFile: false

    - name: Prepare Coverage Action
      if: ${{ github.event.inputs.skiptests == 'false' }}
      working-directory: ./dev/coverage-action
      run: npm install
        
    - name: Process Coverage Stats + Chart
      id: covprocess
      uses: ./dev/coverage-action/
      if: ${{ github.event.inputs.skiptests == 'false' }}
      with:
        token: ${{ github.token }}
        tokenCommon: ${{ secrets.GH_COMMON_TOKEN }}
        repoCommon: common
        version: ${{needs.prepare.outputs.pkg_version}}
        changelog: ${{ steps.changelog.outputs.changes }}
        summary: ${{ github.event.inputs.summary }}
        coverageResultsPath: coverage.json
        histCoveragePath: historical-coverage.json
        
    - name: Create Release
      uses: ncipollo/release-action@v1.11.1
      if: ${{ env.SHOULD_DEPLOY == 'true' && github.event.inputs.dryrun == 'false' }}
      with:
        allowUpdates: true
        draft: false
        tag: ${{ env.PKG_VERSION }}
        name: ${{ env.PKG_VERSION }}
        body: ${{ steps.covprocess.outputs.changelog }}
        artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json"
        token: ${{ secrets.GITHUB_TOKEN }}

    - name: Update Baseline Coverage
      uses: ncipollo/release-action@v1.11.1
      if: ${{ github.event.inputs.updateCoverage == 'true' && github.event.inputs.dryrun == 'false' }}
      with:
        allowUpdates: true
        tag: baseline
        omitBodyDuringUpdate: true
        omitNameDuringUpdate: true
        omitPrereleaseDuringUpdate: true
        replacesArtifacts: true
        artifacts: "coverage.json"
        token: ${{ secrets.GITHUB_TOKEN }}
        
    - name: Upload Build Artifacts
      uses: actions/upload-artifact@v3
      with:
        name: release-${{ env.PKG_VERSION }}
        path: /home/runner/work/release/release.tar.gz
        
    - name: Notify on Slack
      uses: slackapi/slack-github-action@v1.23.0
      with:
        channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
        payload: |
          {
            "text": "Datatracker - Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} completed - <@${{ secrets.SLACK_UID_RJSPARKS }}>"
          }          
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
        
  # -----------------------------------------------------------------
  # SANDBOX
  # -----------------------------------------------------------------
  sandbox:
    name: Deploy to Sandbox
    if: ${{ always() && github.event.inputs.sandbox == 'true' }}
    needs: [prepare, release]
    runs-on: dev-server
    env:
      PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
      
    steps:
    - uses: actions/checkout@v3
        
    - name: Download a Release Artifact
      uses: actions/download-artifact@v3.0.0
      with:
        name: release-${{ env.PKG_VERSION }}
        
    - name: Deploy to containers
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        echo "Reset production flags in settings.py..."
        sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py
        echo "Install Deploy to Container CLI dependencies..."
        cd dev/deploy-to-container
        npm ci
        cd ../..
        echo "Start Deploy..."
        node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain neverusethis.com
                
    - name: Cleanup old docker resources
      env:
        DEBIAN_FRONTEND: noninteractive
      run: |
        docker image prune -a -f