ci: deploy-to-container tool

dev/deploy-to-container/.editorconfig

@@ -0,0 +1,7 @@
+[*]
+indent_size = 2
+indent_style = space
+charset = utf-8
+trim_trailing_whitespace = false
+end_of_line = lf
+insert_final_newline = true

dev/deploy-to-container/.gitignore

@@ -0,0 +1 @@
+/node_modules

dev/deploy-to-container/.npmrc

@@ -0,0 +1,3 @@
+audit = false
+fund = false
+save-exact = true

dev/deploy-to-container/README.md

@@ -0,0 +1,22 @@
+# Datatracker Diff Tool
+
+This tool facilitates testing 2 different datatracker instances (with their own database) and look for changes using the diff tool. Everything runs in docker containers.
+
+The source instance will use the code from where it is run. The target instance can be a remote tag / branch / commmit or another local folder.
+
+## Requirements
+
+- Node `16.x` or later
+- Docker
+
+## Usage
+
+1. From the `dev/diff` directory, run the command:
+```sh
+npm install
+```
+2. Then run the command:
+```sh
+node cli
+```
+3. Follow the on-screen instructions.

dev/deploy-to-container/cli.js

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+
+import Docker from 'dockerode'
+import path from 'path'
+import fs from 'fs-extra'
+import tar from 'tar'
+import yargs from 'yargs/yargs'
+import { hideBin } from 'yargs/helpers'
+import slugify from 'slugify'
+import { nanoid } from 'nanoid'
+
+async function main () {
+  const basePath = process.cwd()
+  const releasePath = path.join(basePath, 'release')
+  const argv = yargs(hideBin(process.argv)).argv
+
+  // Parse branch argument
+  let branch = argv.branch
+  if (!branch) {
+    throw new Error('Missing --branch argument!')
+  }
+  if (branch.indexOf('/') >= 0) {
+    branch = branch.split('/')[1]
+  }
+  branch = slugify(branch, { lower: true, strict: true })
+  if (branch.length < 1) {
+    throw new Error('Branch name is empty!')
+  }
+  console.info(`Will use branch name "${branch}"`)
+
+  // Parse domain argument
+  const domain = argv.domain
+  if (!domain) {
+    throw new Error('Missing --domain argument!')
+  }
+  const hostname = `dt-${branch}.${domain}`
+  console.info(`Will use hostname "${hostname}"`)
+
+  // Connect to Docker Engine API
+  console.info('Connecting to Docker Engine API...')
+  const dock = new Docker()
+  await dock.ping()
+  console.info('Connected to Docker Engine API.')
+
+  // Extract release artifact
+  console.info('Extracting release artifact...')
+  if (!(await fs.pathExists(path.join(basePath, 'release.tar.gz')))) {
+    throw new Error('Missing release.tar.gz file!')
+  }
+  await fs.emptyDir(releasePath)
+  await tar.x({
+    cwd: releasePath,
+    file: 'release.tar.gz'
+  })
+  console.info('Extracted release artifact successfully.')
+
+  // Update the settings_local.py file
+  console.info('Setting configuration files...')
+  const settingsPath = path.join(releasePath, 'ietf/settings_local.py')
+  const cfgRaw = await fs.readFile(path.join(basePath, 'dev/deploy-to-container/settings_local.py'), 'utf8')
+  await fs.outputFile(settingsPath, cfgRaw.replace('__DBHOST__', `dt-db-${branch}`).replace('__SECRETKEY__', nanoid(36)))
+  await fs.copy(path.join(basePath, 'docker/scripts/app-create-dirs.sh'), path.join(releasePath, 'app-create-dirs.sh'))
+  await fs.copy(path.join(basePath, 'dev/deploy-to-container/start.sh'), path.join(releasePath, 'start.sh'))
+  console.info('Updated configuration files.')
+
+  // Pull latest DB image
+  console.info('Pulling latest DB docker image...')
+  const dbImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-db:latest')
+  await new Promise((resolve, reject) => {
+    dock.modem.followProgress(dbImagePullStream, (err, res) => err ? reject(err) : resolve(res))
+  })
+  console.info('Pulled latest DB docker image successfully.')
+  
+  // Pull latest Datatracker Base image
+  console.info('Pulling latest Datatracker base docker image...')
+  const appImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-app-base:latest')
+  await new Promise((resolve, reject) => {
+    dock.modem.followProgress(appImagePullStream, (err, res) => err ? reject(err) : resolve(res))
+  })
+  console.info('Pulled latest Datatracker base docker image.')
+
+  // Terminate existing containers
+  console.info('Ensuring existing containers with same name are terminated...')
+  const containers = await dock.listContainers({ all: true })
+  for (const container of containers) {
+    if (container.Names.includes(`/dt-db-${branch}`) || container.Names.includes(`/dt-app-${branch}`)) {
+      console.info(`Terminating old container ${container.Id}...`)
+      const oldContainer = dock.getContainer(container.Id)
+      if (container.State === 'running') {
+        await oldContainer.stop({ t: 5 })
+      }
+      await oldContainer.remove({ force: true })
+    }
+  }
+  console.info('Existing containers with same name have been terminated.')
+
+  // Create DB container
+  console.info(`Creating DB docker container... [dt-db-${branch}]`)
+  const dbContainer = await dock.createContainer({
+    Image: 'ghcr.io/ietf-tools/datatracker-db:latest',
+    name: `dt-db-${branch}`,
+    Hostname: `dt-db-${branch}`,
+    HostConfig: {
+      NetworkMode: 'bridge',
+      RestartPolicy: {
+        Name: 'unless-stopped'
+      }
+    }
+  })
+  await dbContainer.start()
+  console.info('Created and started DB docker container successfully.')
+
+  // Create Datatracker container
+  console.info(`Creating Datatracker docker container... [dt-app-${branch}]`)
+  const appContainer = await dock.createContainer({
+    Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest',
+    name: `dt-app-${branch}`,
+    Hostname: `dt-app-${branch}`,
+    Env: [
+      `LETSENCRYPT_HOST=${hostname}`,
+      `VIRTUAL_HOST=${hostname}`,
+      `VIRTUAL_PORT=8000`
+    ],
+    HostConfig: {
+      NetworkMode: 'bridge',
+      RestartPolicy: {
+        Name: 'unless-stopped'
+      }
+    },
+    Entrypoint: ['bash', '-c', 'chmod +x ./start.sh && ./start.sh']
+  })
+  console.info(`Created Datatracker docker container successfully.`)
+
+  // Inject updated release into container
+  console.info('Building updated release tarball to inject into container...')
+  const tgzPath = path.join(basePath, 'import.tgz')
+  await tar.c({
+    gzip: true,
+    file: tgzPath,
+    cwd: releasePath,
+    filter (path) {
+      if (path.includes('.git') || path.includes('node_modules')) { return false }
+      return true
+    }
+  }, ['.'])
+  console.info('Injecting archive into Datatracker docker container...')
+  await appContainer.putArchive(tgzPath, {
+    path: '/workspace'
+  })
+  await fs.remove(tgzPath)
+  console.info(`Imported working files into Datatracker docker container successfully.`)
+
+  console.info('Starting Datatracker container...')
+  await appContainer.start()
+  console.info('Datatracker container started successfully.')
+
+  process.exit(0)
+}
+
+main()

dev/deploy-to-container/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "deploy-to-container",
+  "type": "module",
+  "dependencies": {
+    "dockerode": "^3.3.3",
+    "extract-zip": "^2.0.1",
+    "fs-extra": "^10.1.0",
+    "got": "^12.3.1",
+    "lodash-es": "^4.17.21",
+    "luxon": "^3.0.1",
+    "nanoid": "4.0.0",
+    "slugify": "1.6.5",
+    "tar": "^6.1.11",
+    "yargs": "^17.5.1"
+  },
+  "engines": {
+    "node": ">=16"
+  }
+}

dev/deploy-to-container/settings_local.py

@@ -0,0 +1,78 @@
+# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from ietf.settings import *                                          # pyflakes:ignore
+
+ALLOWED_HOSTS = ['*']
+
+DATABASES = {
+    'default': {
+        'HOST': '__DBHOST__',
+        'PORT': 3306,
+        'NAME': 'ietf_utf8',
+        'ENGINE': 'django.db.backends.mysql',
+        'USER': 'django',
+        'PASSWORD': 'RkTkDPFnKpko',
+        'OPTIONS': {
+            'sql_mode': 'STRICT_TRANS_TABLES',
+            'init_command': 'SET storage_engine=InnoDB; SET names "utf8"',
+        },
+    },
+}
+
+SECRET_KEY = "__SECRETKEY__"
+
+DATABASE_TEST_OPTIONS = {
+    'init_command': 'SET storage_engine=InnoDB',
+}
+
+IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits"
+IDSUBMIT_REPOSITORY_PATH = "test/id/"
+IDSUBMIT_STAGING_PATH = "test/staging/"
+INTERNET_DRAFT_ARCHIVE_DIR = "test/archive/"
+INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "test/archive/"
+RFC_PATH = "test/rfc/"
+
+AGENDA_PATH = '/assets/www6s/proceedings/'
+MEETINGHOST_LOGO_PATH = AGENDA_PATH
+
+USING_DEBUG_EMAIL_SERVER=True
+EMAIL_HOST='localhost'
+EMAIL_PORT=2025
+
+MEDIA_BASE_DIR = '/assets'
+MEDIA_ROOT = MEDIA_BASE_DIR + '/media/'
+MEDIA_URL = '/media/'
+
+PHOTOS_DIRNAME = 'photo'
+PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME
+
+SUBMIT_YANG_CATALOG_MODEL_DIR = '/assets/ietf-ftp/yang/catalogmod/'
+SUBMIT_YANG_DRAFT_MODEL_DIR = '/assets/ietf-ftp/yang/draftmod/'
+SUBMIT_YANG_INVAL_MODEL_DIR = '/assets/ietf-ftp/yang/invalmod/'
+SUBMIT_YANG_IANA_MODEL_DIR = '/assets/ietf-ftp/yang/ianamod/'
+SUBMIT_YANG_RFC_MODEL_DIR   = '/assets/ietf-ftp/yang/rfcmod/'
+
+# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker
+import socket
+hostname, _, ips = socket.gethostbyname_ex(socket.gethostname())
+INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips]
+
+# DEV_TEMPLATE_CONTEXT_PROCESSORS = [
+#    'ietf.context_processors.sql_debug',
+# ]
+
+DOCUMENT_PATH_PATTERN = '/assets/ietf-ftp/{doc.type_id}/'
+INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/'
+RFC_PATH = '/assets/ietf-ftp/rfc/'
+CHARTER_PATH = '/assets/ietf-ftp/charter/'
+BOFREQ_PATH = '/assets/ietf-ftp/bofreq/'
+CONFLICT_REVIEW_PATH = '/assets/ietf-ftp/conflict-reviews/'
+STATUS_CHANGE_PATH = '/assets/ietf-ftp/status-changes/'
+INTERNET_DRAFT_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
+INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/assets/ietf-ftp/internet-drafts/'
+
+NOMCOM_PUBLIC_KEYS_DIR = 'data/nomcom_keys/public_keys/'
+SLIDE_STAGING_PATH = 'test/staging/'
+
+DE_GFM_BINARY = '/usr/local/bin/de-gfm'

dev/deploy-to-container/start.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+apt-get update
+apt-get install iputils-ping -y
+ping dt-db-main
+
+# echo "Fixing permissions..."
+# chmod -R 777 ./
+# echo "Ensure all requirements.txt packages are installed..."
+# pip --disable-pip-version-check --no-cache-dir install -r requirements.txt
+# echo "Creating data directories..."
+# chmod +x ./app-create-dirs.sh
+# ./app-create-dirs.sh
+# echo "Running Datatracker checks..."
+# ./ietf/manage.py check
+# echo "Running Datatracker migrations..."
+# ./ietf/manage.py migrate
+# echo "Starting Datatracker..."
+# ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local