datatracker/ietf/meeting/factories.py
Robert Sparks 997239a2ea
feat: write objects to blob storage (#8557)
* feat: basic blobstore infrastructure for dev

* refactor: (broken) attempt to put minio console behind nginx

* feat: initialize blobstore with boto3

* fix: abandon attempt to proxy minio. Use docker compose instead.

* feat: beginning of blob writes

* feat: storage utilities

* feat: test buckets

* chore: black

* chore: remove unused import

* chore: avoid f string when not needed

* fix: inform all settings files about blobstores

* fix: declare types for some settings

* ci: point to new target base

* ci: adjust test workflow

* fix: give the tests debug environment a blobstore

* fix: "better" name declarations

* ci: use devblobstore container

* chore: identify places to write to blobstorage

* chore: remove unreachable code

* feat: store materials

* feat: store statements

* feat: store status changes

* feat: store liaison attachments

* feat: store agendas provided with Interim session requests

* chore: capture TODOs

* feat: store polls and chatlogs

* chore: remove unneeded TODO

* feat: store drafts on submit and post

* fix: handle storage during doc expiration and resurrection

* fix: mirror an unlink

* chore: add/refine TODOs

* feat: store slide submissions

* fix: structure slide test correctly

* fix: correct sense of existence check

* feat: store some indexes

* feat: BlobShadowFileSystemStorage

* feat: shadow floorplans / host logos to the blob

* chore: remove unused import

* feat: strip path from blob shadow names

* feat: shadow photos / thumbs

* refactor: combine photo and photothumb blob kinds

The photos / thumbs were already dropped in the same
directory, so let's not add a distinction at this point.

* style: whitespace

* refactor: use kwargs consistently

* chore: migrations

* refactor: better deconstruct(); rebuild migrations

* fix: use new class in mack patch

* chore: add TODO

* feat: store group index documents

* chore: identify more TODO

* feat: store reviews

* fix: repair merge

* chore: remove unnecessary TODO

* feat: StoredObject metadata

* fix: deburr some debugging code

* fix: only set the deleted timestamp once

* chore: correct typo

* fix: get_or_create vs get and test

* fix: avoid the questionable is_seekable helper

* chore: capture future design consideration

* chore: blob store cfg for k8s

* chore: black

* chore: copyright

* ci: bucket name prefix option + run Black

Adds/uses DATATRACKER_BLOB_STORE_BUCKET_PREFIX option. Other changes
are just Black styling.

* ci: fix typo in bucket name expression

* chore: parameters in app-configure-blobstore

Allows use with other blob stores.

* ci: remove verify=False option

* fix: don't return value from __init__

* feat: option to log timing of S3Storage calls

* chore: units

* fix: deleted->null when storing a file

* style: Black

* feat: log as JSON; refactor to share code; handle exceptions

* ci: add ietf_log_blob_timing option for k8s

* test: --no-manage-blobstore option for running tests

* test: use blob store settings from env, if set

* test: actually set a couple more storage opts

* feat: offswitch (#8541)

* feat: offswitch

* fix: apply ENABLE_BLOBSTORAGE to BlobShadowFileSystemStorage behavior

* chore: log timing of blob reads

* chore: import Config from botocore.config

* chore(deps): import boto3-stubs / botocore

botocore is implicitly imported, but make it explicit
since we refer to it directly

* chore: drop type annotation that mypy loudly ignores

* refactor: add storage methods via mixin

Shares code between Document and DocHistory without
putting it in the base DocumentInfo class, which
lacks the name field. Also makes mypy happy.

* feat: add timeout / retry limit to boto client

* ci: let k8s config the timeouts via env

* chore: repair merge resolution typo

* chore: tweak settings imports

* chore: simplify k8s/settings_local.py imports

---------

Co-authored-by: Jennifer Richards <jennifer@staff.ietf.org>
2025-02-19 17:41:10 -06:00

321 lines
11 KiB
Python

# Copyright The IETF Trust 2016-2022, All Rights Reserved
# -*- coding: utf-8 -*-
import factory
import random
import datetime
from django.core.files.base import ContentFile
from django.db.models import Q
from ietf.doc.storage_utils import store_str
from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule,
TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint,
MeetingHost, ProceedingsMaterial)
from ietf.name.models import (ConstraintName, SessionStatusName, ProceedingsMaterialTypeName,
TimerangeName, SessionPurposeName)
from ietf.doc.factories import ProceedingsMaterialDocFactory
from ietf.group.factories import GroupFactory
from ietf.person.factories import PersonFactory
from ietf.utils.text import xslugify
class MeetingFactory(factory.django.DjangoModelFactory):
class Meta:
model = Meeting
skip_postgeneration_save = True
type_id = factory.Iterator(['ietf','interim'])
city = factory.Faker('city')
country = factory.Faker('country_code')
time_zone = factory.Faker('timezone')
idsubmit_cutoff_day_offset_00 = 13
idsubmit_cutoff_day_offset_01 = 13
idsubmit_cutoff_time_utc = datetime.timedelta(0, 86399)
idsubmit_cutoff_warning_days = datetime.timedelta(days=21)
venue_name = factory.Faker('sentence')
venue_addr = factory.Faker('address')
break_area = factory.Faker('sentence')
reg_area = factory.Faker('sentence')
@factory.lazy_attribute_sequence
def number(self,n):
if self.type_id == 'ietf':
if Meeting.objects.filter(type='ietf').exists():
so_far = max([int(x.number) for x in Meeting.objects.filter(type='ietf')])
return '%02d'%(so_far+1)
else:
return '%02d'%(n+80)
else:
return 'interim-%d-%s-%02d'%(self.date.year,GroupFactory().acronym,n)
@factory.lazy_attribute
def days(self):
if self.type_id == 'ietf':
return 7
else:
return 1
@factory.lazy_attribute
def date(self):
if self.type_id == 'ietf':
num = int(self.number)
year = (num-2)//3+1985
month = ((num-2)%3+1)*4-1
day = random.randint(1,28)
return datetime.date(year, month, day)
else:
return datetime.date(2010,1,1)+datetime.timedelta(days=random.randint(0,3652))
@factory.post_generation
def populate_schedule(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
'''
Create a default schedule, unless the factory is called
with populate_agenda=False
'''
if extracted is None:
extracted = True
if create and extracted:
for x in range(3):
TimeSlotFactory(meeting=obj)
obj.schedule = ScheduleFactory(meeting=obj)
obj.save()
@factory.post_generation
def group_conflicts(obj, create, extracted, **kwargs): # pulint: disable=no-self-argument
"""Add conflict types
Pass a list of ConflictNames as group_conflicts to specify which are enabled.
"""
if extracted is None:
extracted = [
ConstraintName.objects.get(slug=s) for s in [
'chair_conflict', 'tech_overlap', 'key_participant'
]]
if create:
for cn in extracted:
obj.group_conflict_types.add(
cn if isinstance(cn, ConstraintName) else ConstraintName.objects.get(slug=cn)
)
class SessionFactory(factory.django.DjangoModelFactory):
class Meta:
model = Session
skip_postgeneration_save = True
meeting = factory.SubFactory(MeetingFactory)
purpose_id = 'regular'
type_id = 'regular'
group = factory.SubFactory(GroupFactory)
requested_duration = datetime.timedelta(hours=1)
on_agenda = factory.lazy_attribute(lambda obj: SessionPurposeName.objects.get(pk=obj.purpose_id).on_agenda)
has_onsite_tool = factory.lazy_attribute(lambda obj: obj.purpose_id == 'regular')
@factory.post_generation
def status_id(obj, create, extracted, **kwargs):
if create:
if not extracted:
extracted = 'sched'
if extracted not in ['apprw', 'schedw']:
# requested event
SchedulingEvent.objects.create(
session=obj,
status=SessionStatusName.objects.get(slug='schedw'),
by=PersonFactory(),
)
# actual state event
SchedulingEvent.objects.create(
session=obj,
status=SessionStatusName.objects.get(slug=extracted),
by=PersonFactory(),
)
@factory.post_generation
def add_to_schedule(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
'''
Put this session in a timeslot unless the factory is called
with add_to_schedule=False
'''
if extracted is None:
extracted = True
if create and extracted:
ts = obj.meeting.timeslot_set.all()
obj.timeslotassignments.create(timeslot=ts[random.randrange(len(ts))],schedule=obj.meeting.schedule)
class ScheduleFactory(factory.django.DjangoModelFactory):
class Meta:
model = Schedule
meeting = factory.SubFactory(MeetingFactory)
name = factory.Sequence(lambda n: 'schedule_%d'%n)
owner = factory.SubFactory(PersonFactory)
class RoomFactory(factory.django.DjangoModelFactory):
class Meta:
model = Room
skip_postgeneration_save = True
meeting = factory.SubFactory(MeetingFactory)
name = factory.Faker('name')
@factory.post_generation
def session_types(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
"""Prep session types m2m relationship for room, defaulting to 'regular'"""
if create:
session_types = extracted if extracted is not None else ['regular']
for st in session_types:
obj.session_types.add(st)
class TimeSlotFactory(factory.django.DjangoModelFactory):
class Meta:
model = TimeSlot
skip_postgeneration_save = True
meeting = factory.SubFactory(MeetingFactory)
type_id = 'regular'
@factory.post_generation
def location(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument
if create:
if extracted:
# accept the string "none" to set location to null
obj.location = None if extracted == 'none' else extracted
else:
obj.location = RoomFactory(meeting=obj.meeting)
obj.save()
@factory.lazy_attribute
def time(self):
return self.meeting.tz().localize(
datetime.datetime.combine(self.meeting.date, datetime.time(11, 0))
)
@factory.lazy_attribute
def duration(self):
return datetime.timedelta(minutes=30+random.randrange(9)*15)
class SessionPresentationFactory(factory.django.DjangoModelFactory):
class Meta:
model = SessionPresentation
session = factory.SubFactory(SessionFactory)
document = factory.SubFactory('ietf.doc.factories.DocumentFactory')
@factory.lazy_attribute
def rev(self):
return self.document.rev
class FloorPlanFactory(factory.django.DjangoModelFactory):
class Meta:
model = FloorPlan
name = factory.Sequence(lambda n: 'Venue Floor %d' % n)
short = factory.Sequence(lambda n: '%d' % n)
meeting = factory.SubFactory(MeetingFactory)
order = factory.Sequence(lambda n: n)
image = factory.LazyAttribute(
lambda _: ContentFile(
factory.django.ImageField()._make_data(
{'width': 1024, 'height': 768}
), 'floorplan.jpg'
)
)
class SlideSubmissionFactory(factory.django.DjangoModelFactory):
class Meta:
model = SlideSubmission
skip_postgeneration_save = True
session = factory.SubFactory(SessionFactory)
title = factory.Faker('sentence')
filename = factory.Sequence(lambda n: 'test_slide_%d'%n)
submitter = factory.SubFactory(PersonFactory)
make_file = factory.PostGeneration(
lambda obj, create, extracted, **kwargs: open(obj.staged_filepath(),'a').close()
)
store_submission = factory.PostGeneration(
lambda obj, create, extracted, **kwargs: store_str("staging", obj.filename, "")
)
class ConstraintFactory(factory.django.DjangoModelFactory):
class Meta:
model = Constraint
skip_postgeneration_save = True
meeting = factory.SubFactory(MeetingFactory)
source = factory.SubFactory(GroupFactory)
target = factory.SubFactory(GroupFactory)
person = factory.SubFactory(PersonFactory)
time_relation = factory.Iterator(Constraint.TIME_RELATION_CHOICES)
@factory.lazy_attribute
def name(obj):
constraint_list = list(ConstraintName.objects.filter(
Q(slug__in=['bethere','timerange','time_relation','wg_adjacent'])
| Q(meeting=obj.meeting)
))
return random.choice(constraint_list)
@factory.post_generation
def timeranges(self, create, extracted, **kwargs):
if create:
if extracted:
for tr in TimerangeName.objects.filter(slug__in=extracted):
self.timeranges.add(tr)
class MeetingHostFactory(factory.django.DjangoModelFactory):
class Meta:
model = MeetingHost
meeting = factory.SubFactory(MeetingFactory, type_id='ietf')
name = factory.Faker('company')
logo = factory.django.ImageField() # generates an image
def _pmf_doc_name(doc):
"""Helper to generate document name for a ProceedingsMaterialFactory LazyAttribute"""
return 'proceedings-{number}-{slug}'.format(
number=doc.factory_parent.meeting.number,
slug=xslugify(doc.factory_parent.type.slug).replace("_", "-")[:128]
)
class ProceedingsMaterialFactory(factory.django.DjangoModelFactory):
"""Create a ProceedingsMaterial for testing
Note: if you want to specify a type, use type=ProceedingsMaterialTypeName.objects.get(slug='slug')
rather than the type_id='slug' shortcut. The latter will advance the Iterator used to generate
types. This value is then used by the document SubFactory to set the document's title. This will
not match the type of material created.
"""
class Meta:
model = ProceedingsMaterial
meeting = factory.SubFactory(MeetingFactory, type_id='ietf')
type = factory.Iterator(ProceedingsMaterialTypeName.objects.filter(used=True))
# The SelfAttribute atnd LazyAttribute allow the document to be a SubFactory instead
# of a generic LazyAttribute. This allows other attributes on the document to be
# specified as document__external_url, etc.
document = factory.SubFactory(
ProceedingsMaterialDocFactory,
type_id='procmaterials',
title=factory.SelfAttribute('..type.name'),
name=factory.LazyAttribute(_pmf_doc_name),
uploaded_filename=factory.LazyAttribute(
lambda doc: f'{_pmf_doc_name(doc)}-{doc.rev}.pdf'
))
class AttendedFactory(factory.django.DjangoModelFactory):
class Meta:
model = Attended
session = factory.SubFactory(SessionFactory)
person = factory.SubFactory(PersonFactory)