490 lines
17 KiB
YAML
490 lines
17 KiB
YAML
name: Build and Release
|
|
run-name: ${{ github.ref_name == 'release' && '[Prod]' || '[Dev]' }} Build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }}
|
|
|
|
on:
|
|
push:
|
|
branches: [release]
|
|
|
|
workflow_dispatch:
|
|
inputs:
|
|
deploy:
|
|
description: 'Deploy to K8S'
|
|
default: 'Skip'
|
|
required: true
|
|
type: choice
|
|
options:
|
|
- Skip
|
|
- Staging Only
|
|
- Staging + Prod
|
|
sandbox:
|
|
description: 'Deploy to Sandbox'
|
|
default: true
|
|
required: true
|
|
type: boolean
|
|
sandboxNoDbRefresh:
|
|
description: 'Sandbox Disable Daily DB Refresh'
|
|
default: false
|
|
required: true
|
|
type: boolean
|
|
skiptests:
|
|
description: 'Skip Tests'
|
|
default: false
|
|
required: true
|
|
type: boolean
|
|
skiparm:
|
|
description: 'Skip ARM64 Build'
|
|
default: false
|
|
required: true
|
|
type: boolean
|
|
ignoreLowerCoverage:
|
|
description: 'Ignore Lower Coverage'
|
|
default: false
|
|
required: true
|
|
type: boolean
|
|
updateCoverage:
|
|
description: 'Update Baseline Coverage'
|
|
default: false
|
|
required: true
|
|
type: boolean
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# -----------------------------------------------------------------
|
|
# PREPARE
|
|
# -----------------------------------------------------------------
|
|
prepare:
|
|
name: Prepare Release
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
should_deploy: ${{ steps.buildvars.outputs.should_deploy }}
|
|
pkg_version: ${{ steps.buildvars.outputs.pkg_version }}
|
|
from_tag: ${{ steps.semver.outputs.nextStrict }}
|
|
to_tag: ${{ steps.semver.outputs.current }}
|
|
base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 1
|
|
fetch-tags: false
|
|
|
|
- name: Get Next Version (Prod)
|
|
if: ${{ github.ref_name == 'release' }}
|
|
id: semver
|
|
uses: ietf-tools/semver-action@v1
|
|
with:
|
|
token: ${{ github.token }}
|
|
branch: release
|
|
skipInvalidTags: true
|
|
|
|
- name: Get Dev Version
|
|
if: ${{ github.ref_name != 'release' }}
|
|
id: semverdev
|
|
uses: ietf-tools/semver-action@v1
|
|
with:
|
|
token: ${{ github.token }}
|
|
branch: release
|
|
skipInvalidTags: true
|
|
noVersionBumpBehavior: 'current'
|
|
noNewCommitBehavior: 'current'
|
|
|
|
- name: Set Release Flag
|
|
if: ${{ github.ref_name == 'release' }}
|
|
run: |
|
|
echo "IS_RELEASE=true" >> $GITHUB_ENV
|
|
|
|
- name: Create Draft Release
|
|
uses: ncipollo/release-action@v1.14.0
|
|
if: ${{ github.ref_name == 'release' }}
|
|
with:
|
|
prerelease: true
|
|
draft: false
|
|
commit: ${{ github.sha }}
|
|
tag: ${{ steps.semver.outputs.nextStrict }}
|
|
name: ${{ steps.semver.outputs.nextStrict }}
|
|
body: '*pending*'
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Set Build Variables
|
|
id: buildvars
|
|
run: |
|
|
if [[ $IS_RELEASE ]]; then
|
|
echo "Using AUTO SEMVER mode: ${{ steps.semver.outputs.nextStrict }}"
|
|
echo "should_deploy=true" >> $GITHUB_OUTPUT
|
|
echo "pkg_version=${{ steps.semver.outputs.nextStrict }}" >> $GITHUB_OUTPUT
|
|
echo "::notice::Release ${{ steps.semver.outputs.nextStrict }} created using branch $GITHUB_REF_NAME"
|
|
else
|
|
echo "Using TEST mode: ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER"
|
|
echo "should_deploy=false" >> $GITHUB_OUTPUT
|
|
echo "pkg_version=${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT
|
|
echo "::notice::Non-production build ${{ steps.semverdev.outputs.nextMajorStrict }}.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME"
|
|
fi
|
|
|
|
- name: Get Base Image Target Version
|
|
id: baseimgversion
|
|
run: |
|
|
echo "base_image_version=$(sed -n '1p' dev/build/TARGET_BASE)" >> $GITHUB_OUTPUT
|
|
|
|
# -----------------------------------------------------------------
|
|
# TESTS
|
|
# -----------------------------------------------------------------
|
|
|
|
tests:
|
|
name: Run Tests
|
|
uses: ./.github/workflows/tests.yml
|
|
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
|
|
needs: [prepare]
|
|
with:
|
|
ignoreLowerCoverage: ${{ github.event.inputs.ignoreLowerCoverage == 'true' }}
|
|
skipSelenium: true
|
|
targetBaseVersion: ${{ needs.prepare.outputs.base_image_version }}
|
|
|
|
# -----------------------------------------------------------------
|
|
# RELEASE
|
|
# -----------------------------------------------------------------
|
|
release:
|
|
name: Make Release
|
|
if: ${{ !failure() && !cancelled() }}
|
|
needs: [tests, prepare]
|
|
runs-on:
|
|
group: hperf-8c32r
|
|
permissions:
|
|
contents: write
|
|
packages: write
|
|
env:
|
|
SHOULD_DEPLOY: ${{needs.prepare.outputs.should_deploy}}
|
|
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
|
|
FROM_TAG: ${{needs.prepare.outputs.from_tag}}
|
|
TO_TAG: ${{needs.prepare.outputs.to_tag}}
|
|
TARGET_BASE: ${{needs.prepare.outputs.base_image_version}}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 1
|
|
fetch-tags: false
|
|
|
|
- name: Setup Node.js environment
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 18.x
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.x"
|
|
|
|
- name: Download a Coverage Results
|
|
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
|
|
uses: actions/download-artifact@v4.1.8
|
|
with:
|
|
name: coverage
|
|
|
|
- name: Make Release Build
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
BROWSERSLIST_IGNORE_OLD_DATA: 1
|
|
run: |
|
|
echo "PKG_VERSION: $PKG_VERSION"
|
|
echo "GITHUB_SHA: $GITHUB_SHA"
|
|
echo "GITHUB_REF_NAME: $GITHUB_REF_NAME"
|
|
echo "Running frontend build script..."
|
|
echo "Compiling native node packages..."
|
|
yarn rebuild
|
|
echo "Packaging static assets..."
|
|
yarn build --base=https://static.ietf.org/dt/$PKG_VERSION/
|
|
yarn legacy:build
|
|
echo "Setting version $PKG_VERSION..."
|
|
sed -i -r -e "s|^__version__ += '.*'$|__version__ = '$PKG_VERSION'|" ietf/__init__.py
|
|
sed -i -r -e "s|^__release_hash__ += '.*'$|__release_hash__ = '$GITHUB_SHA'|" ietf/__init__.py
|
|
sed -i -r -e "s|^__release_branch__ += '.*'$|__release_branch__ = '$GITHUB_REF_NAME'|" ietf/__init__.py
|
|
|
|
- name: Set Production Flags
|
|
if: ${{ env.SHOULD_DEPLOY == 'true' }}
|
|
run: |
|
|
echo "Setting production flags in settings.py..."
|
|
sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = False/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'production'/" ietf/settings.py
|
|
|
|
- name: Make Release Tarball
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
echo "Build release tarball..."
|
|
mkdir -p /home/runner/work/release
|
|
tar -czf /home/runner/work/release/release.tar.gz -X dev/build/exclude-patterns.txt .
|
|
|
|
- name: Collect + Push Statics
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_STATIC_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_STATIC_KEY_SECRET }}
|
|
AWS_DEFAULT_REGION: auto
|
|
AWS_ENDPOINT_URL: ${{ secrets.CF_R2_ENDPOINT }}
|
|
run: |
|
|
echo "Collecting statics..."
|
|
echo "Using ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }}"
|
|
docker run --rm --name collectstatics -v $(pwd):/workspace ghcr.io/ietf-tools/datatracker-app-base:${{ env.TARGET_BASE }} sh dev/build/collectstatics.sh
|
|
echo "Pushing statics..."
|
|
cd static
|
|
aws s3 sync . s3://static/dt/$PKG_VERSION --only-show-errors
|
|
|
|
- name: Augment dockerignore for docker image build
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
cat >> .dockerignore <<EOL
|
|
.devcontainer
|
|
.github
|
|
.vscode
|
|
k8s
|
|
playwright
|
|
svn-history
|
|
docker-compose.yml
|
|
EOL
|
|
|
|
- name: Setup Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Build Images
|
|
uses: docker/build-push-action@v6
|
|
env:
|
|
DOCKER_BUILD_SUMMARY: false
|
|
with:
|
|
context: .
|
|
file: dev/build/Dockerfile
|
|
platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }}
|
|
push: true
|
|
tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }}
|
|
cache-from: type=gha
|
|
cache-to: type=gha,mode=max
|
|
|
|
- name: Update CHANGELOG
|
|
id: changelog
|
|
uses: Requarks/changelog-action@v1
|
|
if: ${{ env.SHOULD_DEPLOY == 'true' }}
|
|
with:
|
|
token: ${{ github.token }}
|
|
fromTag: ${{ env.FROM_TAG }}
|
|
toTag: ${{ env.TO_TAG }}
|
|
writeToFile: false
|
|
|
|
- name: Download Coverage Results
|
|
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
|
|
uses: actions/download-artifact@v4.1.8
|
|
with:
|
|
name: coverage
|
|
|
|
- name: Prepare Coverage Action
|
|
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
|
|
working-directory: ./dev/coverage-action
|
|
run: npm install
|
|
|
|
- name: Process Coverage Stats + Chart
|
|
id: covprocess
|
|
uses: ./dev/coverage-action/
|
|
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
|
|
with:
|
|
token: ${{ github.token }}
|
|
tokenCommon: ${{ secrets.GH_COMMON_TOKEN }}
|
|
repoCommon: common
|
|
version: ${{needs.prepare.outputs.pkg_version}}
|
|
changelog: ${{ steps.changelog.outputs.changes }}
|
|
summary: ''
|
|
coverageResultsPath: coverage.json
|
|
histCoveragePath: historical-coverage.json
|
|
|
|
- name: Create Release
|
|
uses: ncipollo/release-action@v1.14.0
|
|
if: ${{ env.SHOULD_DEPLOY == 'true' }}
|
|
with:
|
|
allowUpdates: true
|
|
makeLatest: true
|
|
draft: false
|
|
tag: ${{ env.PKG_VERSION }}
|
|
name: ${{ env.PKG_VERSION }}
|
|
body: ${{ steps.covprocess.outputs.changelog }}
|
|
artifacts: "/home/runner/work/release/release.tar.gz,coverage.json,historical-coverage.json"
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Update Baseline Coverage
|
|
uses: ncipollo/release-action@v1.14.0
|
|
if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }}
|
|
with:
|
|
allowUpdates: true
|
|
tag: baseline
|
|
omitBodyDuringUpdate: true
|
|
omitNameDuringUpdate: true
|
|
omitPrereleaseDuringUpdate: true
|
|
replacesArtifacts: true
|
|
artifacts: "coverage.json"
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Upload Build Artifacts
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: release-${{ env.PKG_VERSION }}
|
|
path: /home/runner/work/release/release.tar.gz
|
|
|
|
# -----------------------------------------------------------------
|
|
# NOTIFY
|
|
# -----------------------------------------------------------------
|
|
notify:
|
|
name: Notify
|
|
if: ${{ always() }}
|
|
needs: [prepare, tests, release]
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
|
|
|
|
steps:
|
|
- name: Notify on Slack (Success)
|
|
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
|
|
uses: slackapi/slack-github-action@v1.27.0
|
|
with:
|
|
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
|
|
payload: |
|
|
{
|
|
"text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
|
|
"attachments": [
|
|
{
|
|
"color": "28a745",
|
|
"fields": [
|
|
{
|
|
"title": "Status",
|
|
"short": true,
|
|
"value": "Completed"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
|
|
- name: Notify on Slack (Failure)
|
|
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
|
|
uses: slackapi/slack-github-action@v1.27.0
|
|
with:
|
|
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
|
|
payload: |
|
|
{
|
|
"text": "Datatracker Build <https://github.com/ietf-tools/datatracker/actions/runs/${{ github.run_id }}|${{ env.PKG_VERSION }}> by ${{ github.triggering_actor }} - <@${{ secrets.SLACK_UID_RJSPARKS }}>",
|
|
"attachments": [
|
|
{
|
|
"color": "a82929",
|
|
"fields": [
|
|
{
|
|
"title": "Status",
|
|
"short": true,
|
|
"value": "Failed"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
|
|
|
|
# -----------------------------------------------------------------
|
|
# SANDBOX
|
|
# -----------------------------------------------------------------
|
|
sandbox:
|
|
name: Deploy to Sandbox
|
|
if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }}
|
|
needs: [prepare, release]
|
|
runs-on: [self-hosted, dev-server]
|
|
environment:
|
|
name: sandbox
|
|
env:
|
|
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Download a Release Artifact
|
|
uses: actions/download-artifact@v4.1.8
|
|
with:
|
|
name: release-${{ env.PKG_VERSION }}
|
|
|
|
- name: Deploy to containers
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
echo "Reset production flags in settings.py..."
|
|
sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py
|
|
echo "Install Deploy to Container CLI dependencies..."
|
|
cd dev/deploy-to-container
|
|
npm ci
|
|
cd ../..
|
|
echo "Start Deploy..."
|
|
node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }}
|
|
|
|
- name: Cleanup old docker resources
|
|
env:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
run: |
|
|
docker image prune -a -f
|
|
|
|
# -----------------------------------------------------------------
|
|
# STAGING
|
|
# -----------------------------------------------------------------
|
|
staging:
|
|
name: Deploy to Staging
|
|
if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging Only' || github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }}
|
|
needs: [prepare, release]
|
|
runs-on: ubuntu-latest
|
|
environment:
|
|
name: staging
|
|
env:
|
|
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
|
|
|
|
steps:
|
|
- name: Deploy to staging
|
|
uses: the-actions-org/workflow-dispatch@v4
|
|
with:
|
|
workflow: deploy.yml
|
|
repo: ietf-tools/infra-k8s
|
|
ref: main
|
|
token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
|
|
inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }'
|
|
wait-for-completion: true
|
|
wait-for-completion-timeout: 10m
|
|
wait-for-completion-interval: 30s
|
|
display-workflow-run-url: false
|
|
|
|
# -----------------------------------------------------------------
|
|
# PROD
|
|
# -----------------------------------------------------------------
|
|
prod:
|
|
name: Deploy to Production
|
|
if: ${{ !failure() && !cancelled() && (github.event.inputs.deploy == 'Staging + Prod' || github.ref_name == 'release') }}
|
|
needs: [prepare, staging]
|
|
runs-on: ubuntu-latest
|
|
environment:
|
|
name: production
|
|
env:
|
|
PKG_VERSION: ${{needs.prepare.outputs.pkg_version}}
|
|
|
|
steps:
|
|
- name: Deploy to production
|
|
uses: the-actions-org/workflow-dispatch@v4
|
|
with:
|
|
workflow: deploy.yml
|
|
repo: ietf-tools/infra-k8s
|
|
ref: main
|
|
token: ${{ secrets.GH_INFRA_K8S_TOKEN }}
|
|
inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }'
|
|
wait-for-completion: true
|
|
wait-for-completion-timeout: 10m
|
|
wait-for-completion-interval: 30s
|
|
display-workflow-run-url: false
|