datatracker/docker-compose.yml
Robert Sparks 997239a2ea
feat: write objects to blob storage (#8557)
* feat: basic blobstore infrastructure for dev

* refactor: (broken) attempt to put minio console behind nginx

* feat: initialize blobstore with boto3

* fix: abandon attempt to proxy minio. Use docker compose instead.

* feat: beginning of blob writes

* feat: storage utilities

* feat: test buckets

* chore: black

* chore: remove unused import

* chore: avoid f string when not needed

* fix: inform all settings files about blobstores

* fix: declare types for some settings

* ci: point to new target base

* ci: adjust test workflow

* fix: give the tests debug environment a blobstore

* fix: "better" name declarations

* ci: use devblobstore container

* chore: identify places to write to blobstorage

* chore: remove unreachable code

* feat: store materials

* feat: store statements

* feat: store status changes

* feat: store liaison attachments

* feat: store agendas provided with Interim session requests

* chore: capture TODOs

* feat: store polls and chatlogs

* chore: remove unneeded TODO

* feat: store drafts on submit and post

* fix: handle storage during doc expiration and resurrection

* fix: mirror an unlink

* chore: add/refine TODOs

* feat: store slide submissions

* fix: structure slide test correctly

* fix: correct sense of existence check

* feat: store some indexes

* feat: BlobShadowFileSystemStorage

* feat: shadow floorplans / host logos to the blob

* chore: remove unused import

* feat: strip path from blob shadow names

* feat: shadow photos / thumbs

* refactor: combine photo and photothumb blob kinds

The photos / thumbs were already dropped in the same
directory, so let's not add a distinction at this point.

* style: whitespace

* refactor: use kwargs consistently

* chore: migrations

* refactor: better deconstruct(); rebuild migrations

* fix: use new class in mack patch

* chore: add TODO

* feat: store group index documents

* chore: identify more TODO

* feat: store reviews

* fix: repair merge

* chore: remove unnecessary TODO

* feat: StoredObject metadata

* fix: deburr some debugging code

* fix: only set the deleted timestamp once

* chore: correct typo

* fix: get_or_create vs get and test

* fix: avoid the questionable is_seekable helper

* chore: capture future design consideration

* chore: blob store cfg for k8s

* chore: black

* chore: copyright

* ci: bucket name prefix option + run Black

Adds/uses DATATRACKER_BLOB_STORE_BUCKET_PREFIX option. Other changes
are just Black styling.

* ci: fix typo in bucket name expression

* chore: parameters in app-configure-blobstore

Allows use with other blob stores.

* ci: remove verify=False option

* fix: don't return value from __init__

* feat: option to log timing of S3Storage calls

* chore: units

* fix: deleted->null when storing a file

* style: Black

* feat: log as JSON; refactor to share code; handle exceptions

* ci: add ietf_log_blob_timing option for k8s

* test: --no-manage-blobstore option for running tests

* test: use blob store settings from env, if set

* test: actually set a couple more storage opts

* feat: offswitch (#8541)

* feat: offswitch

* fix: apply ENABLE_BLOBSTORAGE to BlobShadowFileSystemStorage behavior

* chore: log timing of blob reads

* chore: import Config from botocore.config

* chore(deps): import boto3-stubs / botocore

botocore is implicitly imported, but make it explicit
since we refer to it directly

* chore: drop type annotation that mypy loudly ignores

* refactor: add storage methods via mixin

Shares code between Document and DocHistory without
putting it in the base DocumentInfo class, which
lacks the name field. Also makes mypy happy.

* feat: add timeout / retry limit to boto client

* ci: let k8s config the timeouts via env

* chore: repair merge resolution typo

* chore: tweak settings imports

* chore: simplify k8s/settings_local.py imports

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
2025-02-19 17:41:10 -06:00

119 lines
3.4 KiB
YAML

services:
app:
build:
context: .
dockerfile: docker/app.Dockerfile
init: true
# Overrides default command so things don't shut down after the process ends.
command: sleep infinity
# Runs app on the same network as the database container, allows "forwardPorts" in devcontainer.json function.
# network_mode: service:db
depends_on:
- db
- mq
- blobstore
ipc: host
environment:
DISPLAY: host.docker.internal:0
# USER: django
# UID: 1001
# GID: 1001
# DATADIR: data
# DJANGO_SETTINGS_MODULE: settings_test
# Uncomment the next line to use a non-root user for all processes.
# user: dev
# Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
# (Adding the "ports" property to this file will not forward from a Codespace.)
db:
image: ghcr.io/ietf-tools/datatracker-db:latest
# build:
# context: ..
# dockerfile: docker/db.Dockerfile
restart: unless-stopped
volumes:
- postgresdb-data:/var/lib/postgresql/data
# Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally.
# (Adding the "ports" property to this file will not forward from a Codespace.)
pgadmin:
image: dpage/pgadmin4:latest
restart: unless-stopped
environment:
- PGADMIN_DEFAULT_EMAIL=dev@ietf.org
- PGADMIN_DEFAULT_PASSWORD=dev
- PGADMIN_CONFIG_LOGIN_BANNER="Login with dev@ietf.org / dev"
- PGADMIN_DISABLE_POSTFIX=True
- PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=False
- SCRIPT_NAME=/pgadmin
volumes:
- ./docker/configs/pgadmin-servers.json:/pgadmin4/servers.json
static:
image: ghcr.io/ietf-tools/static:latest
restart: unless-stopped
mq:
image: rabbitmq:3-alpine
restart: unless-stopped
celery:
image: ghcr.io/ietf-tools/datatracker-celery:latest
init: true
environment:
CELERY_APP: ietf
CELERY_ROLE: worker
UPDATE_REQUIREMENTS_FROM: requirements.txt
DEV_MODE: "yes"
command:
- '--loglevel=INFO'
depends_on:
- db
restart: unless-stopped
stop_grace_period: 1m
volumes:
- .:/workspace
- app-assets:/assets
blobstore:
image: ghcr.io/ietf-tools/datatracker-devblobstore:latest
restart: unless-stopped
volumes:
- "minio-data:/data"
# Celery Beat is a periodic task runner. It is not normally needed for development,
# but can be enabled by uncommenting the following.
#
# beat:
# image: ghcr.io/ietf-tools/datatracker-celery:latest
# init: true
# environment:
# CELERY_APP: ietf
# CELERY_ROLE: beat
# UPDATE_REQUIREMENTS_FROM: requirements.txt
# command:
# - '--loglevel=INFO'
# depends_on:
# - db
# restart: unless-stopped
# stop_grace_period: 1m
# volumes:
# - .:/workspace
# - app-assets:/assets
volumes:
postgresdb-data:
app-assets:
minio-data: