Allow generated schedules to inherit from a base schedule. Fixes #3170. Commit ready for merge.

- Legacy-Id: 19297

ietf/meeting/management/commands/generate_schedule.py

@@ -13,6 +13,7 @@ import time
 
 from collections import defaultdict
 from functools import lru_cache
+from typing import NamedTuple, Optional
 
 from django.contrib.humanize.templatetags.humanize import intcomma
 from django.core.management.base import BaseCommand, CommandError
@@ -22,6 +23,7 @@ import debug                            # pyflakes:ignore
 
 from ietf.person.models import Person
 from ietf.meeting import models
+from ietf.meeting.helpers import get_person_by_email
 
 # 40 runs of the optimiser for IETF 106 with cycles=160 resulted in 16
 # zero-violation invocations, with a mean number of runs of 91 and 
@@ -32,6 +34,24 @@ from ietf.meeting import models
 OPTIMISER_MAX_CYCLES = 160
 
 
+class ScheduleId(NamedTuple):
+    """Represents a schedule id as name and owner"""
+    name: str
+    owner: Optional[str] = None
+
+    @classmethod
+    def from_str(cls, s):
+        """Parse id of the form [owner/]name"""
+        return cls(*reversed(s.split('/', 1)))
+
+    @classmethod
+    def from_schedule(cls, sched):
+        return cls(sched.name, str(sched.owner.email()))
+
+    def __str__(self):
+        return '/'.join(tok for tok in reversed(self) if tok is not None)
+
+
 class Command(BaseCommand):
     help = 'Create a meeting schedule'
 
@@ -43,13 +63,22 @@ class Command(BaseCommand):
         parser.add_argument('-r', '--max-runs', type=int, dest='max_cycles',
                             default=OPTIMISER_MAX_CYCLES,
                             help='maximum optimiser runs')
+        parser.add_argument('-b', '--base-schedule',
+                            type=ScheduleId.from_str,
+                            dest='base_id',
+                            default=None,
+                            help=(
+                                'Base schedule for generated schedule, specified as "[owner/]name"'
+                                ' (default is no base schedule; owner not required if name is unique)'
+                            ))
 
-    def handle(self, meeting, name, max_cycles, verbosity, *args, **kwargs):
+    def handle(self, meeting, name, max_cycles, verbosity, base_id, *args, **kwargs):
-        ScheduleHandler(self.stdout, meeting, name, max_cycles, verbosity).run()
+        ScheduleHandler(self.stdout, meeting, name, max_cycles, verbosity, base_id).run()
 
 
 class ScheduleHandler(object):
-    def __init__(self, stdout, meeting_number, name=None, max_cycles=OPTIMISER_MAX_CYCLES, verbosity=1):
+    def __init__(self, stdout, meeting_number, name=None, max_cycles=OPTIMISER_MAX_CYCLES,
+                 verbosity=1, base_id=None):
         self.stdout = stdout
         self.verbosity = verbosity
         self.name = name
@@ -61,14 +90,32 @@ class ScheduleHandler(object):
                 raise CommandError('Unknown meeting number {}'.format(meeting_number))
         else:
             self.meeting = models.Meeting.get_current_meeting()
+
+        if base_id is None:
+            self.base_schedule = None
+        else:
+            base_candidates = models.Schedule.objects.filter(meeting=self.meeting, name=base_id.name)
+            if base_id.owner is not None:
+                base_candidates = base_candidates.filter(owner=get_person_by_email(base_id.owner))
+            if base_candidates.count() == 0:
+                raise CommandError('Base schedule "{}" not found'.format(base_id))
+            elif base_candidates.count() >= 2:
+                raise CommandError('Base schedule "{}" not unique (candidates are {})'.format(
+                    base_id,
+                    ', '.join(str(ScheduleId.from_schedule(sched)) for sched in base_candidates)
+                ))
+            else:
+                self.base_schedule = base_candidates.first()  # only have one
+
         if self.verbosity >= 1:
-            self.stdout.write("\nRunning automatic schedule layout for meeting IETF %s\n\n" % self.meeting.number)
+            msgs = ['Running automatic schedule layout for meeting IETF {}'.format(self.meeting.number)]
+            if self.base_schedule is not None:
+                msgs.append('Applying schedule {} as base schedule'.format(ScheduleId.from_schedule(self.base_schedule)))
+            self.stdout.write('\n{}\n\n'.format('\n'.join(msgs)))
         self._load_meeting()
 
     def run(self):
         """Schedule all sessions"""
-
-
         beg_time = time.time()
         self.schedule.fill_initial_schedule()
         violations, cost = self.schedule.total_schedule_cost()
@@ -107,6 +154,7 @@ class ScheduleHandler(object):
         schedule_db = models.Schedule.objects.create(
             meeting=self.meeting,
             name=self.name,
+            base=self.base_schedule,
             owner=Person.objects.get(name='(System)'),
             public=False,
             visible=True,
@@ -114,40 +162,89 @@ class ScheduleHandler(object):
         )
         self.schedule.save_assignments(schedule_db)
         self.stdout.write('Schedule saved as {}'.format(self.name))
-        
+
+    def _available_timeslots(self):
+        """Find timeslots available for schedule generation
+
+        Excludes:
+          * sunday timeslots
+          *  timeslots used by the base schedule, if any
+        """
+        # n.b., models.TimeSlot is not the same as TimeSlot!
+        timeslots_db = models.TimeSlot.objects.filter(
+            meeting=self.meeting,
+            type_id='regular',
+        ).exclude(
+            location__capacity=None,
+        )
+
+        if self.base_schedule is None:
+            fixed_timeslots = models.TimeSlot.objects.none()
+        else:
+            fixed_timeslots = timeslots_db.filter(pk__in=self.base_schedule.qs_timeslots_in_use())
+        free_timeslots = timeslots_db.exclude(pk__in=fixed_timeslots)
+
+        timeslots = {TimeSlot(t, self.verbosity) for t in free_timeslots.select_related('location')}
+        timeslots.update(
+            TimeSlot(t, self.verbosity, is_fixed=True) for t in fixed_timeslots.select_related('location')
+        )
+        return {t for t in timeslots if t.day != 'sunday'}
+
+    def _sessions_to_schedule(self, *args, **kwargs):
+        """Find sessions that need to be scheduled
+
+        Extra arguments are passed to the Session constructor.
+        """
+        sessions_db = models.Session.objects.filter(
+            meeting=self.meeting,
+            type_id='regular',
+            schedulingevent__status_id='schedw',
+        )
+
+        if self.base_schedule is None:
+            fixed_sessions = models.Session.objects.none()
+        else:
+            fixed_sessions = sessions_db.filter(pk__in=self.base_schedule.qs_sessions_scheduled())
+        free_sessions = sessions_db.exclude(pk__in=fixed_sessions)
+
+        sessions = {
+            Session(self.stdout, self.meeting, s, is_fixed=False, *args, **kwargs)
+            for s in free_sessions.select_related('group')
+        }
+
+        sessions.update({
+            Session(self.stdout, self.meeting, s, is_fixed=True, *args, **kwargs)
+            for s in fixed_sessions.select_related('group')
+        })
+        return sessions
+
     def _load_meeting(self):
         """Load all timeslots and sessions into in-memory objects."""
         business_constraint_costs = {
             bc.slug: bc.penalty
             for bc in models.BusinessConstraint.objects.all()
         }
-        
+
-        timeslots_db = models.TimeSlot.objects.filter(
+        timeslots = self._available_timeslots()
-            meeting=self.meeting,
-            type_id='regular',
-        ).exclude(location__capacity=None).select_related('location')
-        
-        timeslots = {TimeSlot(t, self.verbosity) for t in timeslots_db}
-        timeslots = {t for t in timeslots if t.day != 'sunday'}
         for timeslot in timeslots:
             timeslot.store_relations(timeslots)
 
-        sessions_db = models.Session.objects.filter(
+        sessions = self._sessions_to_schedule(business_constraint_costs, self.verbosity)
-            meeting=self.meeting,
-            type_id='regular',
-            schedulingevent__status_id='schedw',
-        ).select_related('group')
-        
-        sessions = {Session(self.stdout, self.meeting, s, business_constraint_costs, self.verbosity)
-                    for s in sessions_db}
         for session in sessions:
             # The complexity of a session also depends on how many
             # sessions have declared a conflict towards this session.
             session.update_complexity(sessions)
 
         self.schedule = Schedule(
-            self.stdout, timeslots, sessions, business_constraint_costs, self.max_cycles, self.verbosity)
+            self.stdout,
-        self.schedule.adjust_for_timeslot_availability()
+            timeslots,
+            sessions,
+            business_constraint_costs,
+            self.max_cycles,
+            self.verbosity,
+            self.base_schedule,
+        )
+        self.schedule.adjust_for_timeslot_availability()  # calculates some fixed costs
 
 
 class Schedule(object):
@@ -156,19 +253,79 @@ class Schedule(object):
     The schedule is internally represented as a dict, timeslots being keys, sessions being values.
     Note that "timeslot" means the combination of a timeframe and a location.
     """
-    def __init__(self, stdout, timeslots, sessions, business_constraint_costs, max_cycles, verbosity):
+    def __init__(self, stdout, timeslots, sessions, business_constraint_costs,
+                 max_cycles, verbosity, base_schedule=None):
         self.stdout = stdout
         self.timeslots = timeslots
-        self.sessions = sessions
+        self.sessions = sessions or []
         self.business_constraint_costs = business_constraint_costs
         self.verbosity = verbosity
         self.schedule = dict()
         self.best_cost = math.inf
         self.best_schedule = None
-        self.fixed_cost = 0
+        self._fixed_costs = dict()  # key = type of cost
-        self.fixed_violations = []
+        self._fixed_violations = dict()  # key = type of cost
         self.max_cycles = max_cycles
-        
+        self.base_schedule = self._load_base_schedule(base_schedule) if base_schedule else None
+
+    def __str__(self):
+        return 'Schedule ({} timeslots, {} sessions, {} scheduled, {} in base schedule)'.format(
+            len(self.timeslots),
+            len(self.sessions),
+            len(self.schedule),
+            len(self.base_schedule) if self.base_schedule else 0,
+        )
+
+    def pretty_print(self, include_base=True):
+        """Pretty print the schedule"""
+        last_day = None
+        sched = dict(self.schedule)
+        if include_base:
+            sched.update(self.base_schedule)
+        for slot in sorted(sched, key=lambda ts: ts.start):
+            if last_day != slot.start.date():
+                last_day = slot.start.date()
+                print("""
+-----------------
+ Day: {}
+-----------------""".format(slot.start.date()))
+
+            print('{}: {}{}'.format(
+                models.TimeSlot.objects.get(pk=slot.timeslot_pk),
+                models.Session.objects.get(pk=sched[slot].session_pk),
+                ' [BASE]' if slot in self.base_schedule else '',
+            ))
+
+    @property
+    def fixed_cost(self):
+        return sum(self._fixed_costs.values())
+
+    @property
+    def fixed_violations(self):
+        return sum(self._fixed_violations.values(), [])
+
+    def add_fixed_cost(self, label, violations, cost):
+        self._fixed_costs[label] = cost
+        self._fixed_violations[label] = violations
+
+    @property
+    def free_sessions(self):
+        """Sessions that can be moved by the schedule"""
+        return (sess for sess in self.sessions if not sess.is_fixed)
+
+    @property
+    def free_timeslots(self):
+        """Timeslots that can be filled by the schedule"""
+        return (t for t in self.timeslots if not t.is_fixed)
+
+    def _load_base_schedule(self, db_base_schedule):
+        session_lut = {s.session_pk: s for s in self.sessions}
+        timeslot_lut = {t.timeslot_pk: t for t in self.timeslots}
+        base_schedule = dict()
+        for assignment in db_base_schedule.assignments.filter(session__in=session_lut, timeslot__in=timeslot_lut):
+            base_schedule[timeslot_lut[assignment.timeslot.pk]] = session_lut[assignment.session.pk]
+        return base_schedule
+
     def save_assignments(self, schedule_db):
         for timeslot, session in self.schedule.items():
             models.SchedTimeSessAssignment.objects.create(
@@ -188,14 +345,17 @@ class Schedule(object):
         of trimming in advance is to prevent the optimiser from trying to resolve
         a constraint that can never be resolved.
         """
-        if len(self.sessions) > len(self.timeslots):
+        num_to_schedule = len(list(self.free_sessions))
+        num_free_timeslots = len(list(self.free_timeslots))
+        if  num_to_schedule > num_free_timeslots:
             raise CommandError('More sessions ({}) than timeslots ({})'
-                               .format(len(self.sessions), len(self.timeslots)))
+                               .format(num_to_schedule, num_free_timeslots))
     
         def make_capacity_adjustments(t_attr, s_attr):
-            availables = [getattr(timeslot, t_attr) for timeslot in self.timeslots]
+            availables = [getattr(timeslot, t_attr) for timeslot in self.free_timeslots]
             availables.sort()
-            sessions = sorted(self.sessions, key=lambda s: getattr(s, s_attr), reverse=True)
+            sessions = sorted(self.free_sessions, key=lambda s: getattr(s, s_attr), reverse=True)
+            violations, cost = [], 0
             for session in sessions:
                 found_fit = False
                 for idx, available in enumerate(availables):
@@ -209,12 +369,21 @@ class Schedule(object):
                     msg = f.format(t_attr, session.group, getattr(session, s_attr), largest_available)
                     setattr(session, s_attr, largest_available)
                     availables.pop(-1)
-                    self.fixed_cost += self.business_constraint_costs['session_requires_trim']
+                    cost += self.business_constraint_costs['session_requires_trim']
-                    self.fixed_violations.append(msg)
+                    violations.append(msg)
-        
+            return violations, cost
-        make_capacity_adjustments('duration', 'requested_duration')
+
-        make_capacity_adjustments('capacity', 'attendees')
+        self.add_fixed_cost(
-                
+            'session_requires_duration_trim',
+            *make_capacity_adjustments('duration', 'requested_duration'),
+        )
+
+        self.add_fixed_cost(
+            'session_requires_capacity_trim',
+            *make_capacity_adjustments('capacity', 'attendees'),
+        )
+
+
     def total_schedule_cost(self):
         """
         Calculate the total cost of the current schedule in self.schedule.
@@ -224,11 +393,15 @@ class Schedule(object):
         Returns a tuple of violations (list of strings) and the total cost (integer). 
         """
         violations, cost = self.calculate_dynamic_cost()
+        if self.base_schedule is not None:
+            # Include dynamic costs from the base schedule as a fixed cost for the generated schedule.
+            # Fixed costs from the base schedule are included in the costs computed by adjust_for_timeslot_availability.
+            self.add_fixed_cost('base_schedule', *self.calculate_dynamic_cost(self.base_schedule, include_fixed=True))
         violations += self.fixed_violations
         cost += self.fixed_cost
         return violations, cost
 
-    def calculate_dynamic_cost(self, schedule=None):
+    def calculate_dynamic_cost(self, schedule=None, include_fixed=False):
         """
         Calculate the dynamic cost of the current schedule in self.schedule,
         or a different provided schedule. "Dynamic" cost means these are costs
@@ -237,18 +410,23 @@ class Schedule(object):
         """
         if not schedule:
             schedule = self.schedule
+        if self.base_schedule is not None:
+            schedule = dict(schedule)  # make a copy
+            schedule.update(self.base_schedule)
+
         violations, cost = [], 0
         
         # For performance, a few values are pre-calculated in bulk
         group_sessions = defaultdict(set)
         overlapping_sessions = defaultdict(set)
         for timeslot, session in schedule.items():
-            group_sessions[session.group].add((timeslot, session))
+            group_sessions[session.group].add((timeslot, session))  # (timeslot, session), not just session!
-            overlapping_sessions[timeslot].update({schedule.get(t) for t in timeslot.overlaps})
+            overlapping_sessions[timeslot].update({schedule[t] for t in timeslot.overlaps if t in schedule})
             
         for timeslot, session in schedule.items():
             session_violations, session_cost = session.calculate_cost(
-                schedule, timeslot, overlapping_sessions[timeslot], group_sessions[session.group])
+                schedule, timeslot, overlapping_sessions[timeslot], group_sessions[session.group], include_fixed
+            )
             violations += session_violations
             cost += session_cost
 
@@ -270,11 +448,11 @@ class Schedule(object):
         """
         if self.verbosity >= 2:
             self.stdout.write('== Initial scheduler starting, scheduling {} sessions in {} timeslots =='
-                              .format(len(self.sessions), len(self.timeslots)))
+                              .format(len(list(self.free_sessions)), len(list(self.free_timeslots))))
-        sessions = sorted(self.sessions, key=lambda s: s.complexity, reverse=True)
+        sessions = sorted(self.free_sessions, key=lambda s: s.complexity, reverse=True)
 
         for session in sessions:
-            possible_slots = [t for t in self.timeslots if t not in self.schedule.keys()]
+            possible_slots = [t for t in self.free_timeslots if t not in self.schedule.keys()]
             random.shuffle(possible_slots)
             
             def timeslot_preference(t):
@@ -326,6 +504,8 @@ class Schedule(object):
                 self._shuffle_conflicted_sessions(items)
 
             for original_timeslot, session in items:
+                if session.is_fixed:
+                    continue
                 best_cost = self.calculate_dynamic_cost()[1]
                 if best_cost == 0:
                     if self.verbosity >= 1 and self.stdout.isatty():
@@ -336,7 +516,7 @@ class Schedule(object):
                     return run_count
                 best_timeslot = None
 
-                for possible_new_slot in self.timeslots:
+                for possible_new_slot in self.free_timeslots:
                     cost = self._cost_for_switch(original_timeslot, possible_new_slot)
                     if cost < best_cost:
                         best_cost = cost
@@ -382,8 +562,7 @@ class Schedule(object):
                               .format(', '.join([s.group for t, s in to_reschedule])))
         
         for original_timeslot, rescheduling_session in to_reschedule:
-            possible_new_slots = list(self.timeslots)
+            possible_new_slots = list(t for t in self.free_timeslots if t != original_timeslot)
-            possible_new_slots.remove(original_timeslot)
             random.shuffle(possible_new_slots)
             
             for possible_new_slot in possible_new_slots:
@@ -406,7 +585,7 @@ class Schedule(object):
         """
         optimised_timeslots = set()
         for timeslot in list(self.schedule.keys()):
-            if timeslot in optimised_timeslots:
+            if timeslot in optimised_timeslots or timeslot.is_fixed:
                 continue
             timeslot_overlaps = sorted(timeslot.full_overlaps, key=lambda t: t.capacity, reverse=True)
             sessions_overlaps = [self.schedule.get(t) for t in timeslot_overlaps]
@@ -414,6 +593,8 @@ class Schedule(object):
             assert len(timeslot_overlaps) == len(sessions_overlaps)
             
             for new_timeslot in timeslot_overlaps:
+                if new_timeslot.is_fixed:
+                    continue
                 new_session = sessions_overlaps.pop(0)
                 if not new_session and new_timeslot in self.schedule:
                     del self.schedule[new_timeslot]
@@ -482,9 +663,10 @@ class TimeSlot(object):
     This TimeSlot class is analogous to the TimeSlot class in the models,
     i.e. it represents a timeframe in a particular location.
     """
-    def __init__(self, timeslot_db, verbosity):
+    def __init__(self, timeslot_db, verbosity, is_fixed=False):
         """Initialise this object from a TimeSlot model instance."""
         self.verbosity = verbosity
+        self.is_fixed = is_fixed
         self.timeslot_pk = timeslot_db.pk
         self.location_pk = timeslot_db.location.pk
         self.capacity = timeslot_db.location.capacity
@@ -534,7 +716,7 @@ class Session(object):
     i.e. it represents a single session to be scheduled. It also pulls
     in data about constraints, group parents, etc.
     """
-    def __init__(self, stdout, meeting, session_db, business_constraint_costs, verbosity):
+    def __init__(self, stdout, meeting, session_db, business_constraint_costs, verbosity, is_fixed=False):
         """
         Initialise this object from a Session model instance.
         This includes collecting all constraints from the database,
@@ -555,6 +737,7 @@ class Session(object):
         ])
         self.is_bof = session_db.group.state_id == 'bof'
         self.is_prg = session_db.group.type_id == 'rg' and session_db.group.state_id == 'proposed'
+        self.is_fixed = is_fixed  # if True, cannot be moved
 
         self.attendees = session_db.attendees
         if not self.attendees:
@@ -629,7 +812,7 @@ class Session(object):
     def fits_in_timeslot(self, timeslot):
         return self.attendees <= timeslot.capacity and self.requested_duration <= timeslot.duration
 
-    def calculate_cost(self, schedule, my_timeslot, overlapping_sessions, my_sessions):
+    def calculate_cost(self, schedule, my_timeslot, overlapping_sessions, my_sessions, include_fixed=False):
         """
         Calculate the cost of this session, in the provided schedule, with this session
         being in my_timeslot, and a given set of overlapping sessions and the set of
@@ -642,20 +825,25 @@ class Session(object):
         The return value is a tuple of violations (list of strings) and a cost (integer).        
         """
         violations, cost = [], 0
-        overlapping_sessions = tuple(overlapping_sessions)
+        # Ignore overlap between two fixed sessions when calculating dynamic cost
-        
+        overlapping_sessions = tuple(
-        if self.attendees > my_timeslot.capacity:
+            o for o in overlapping_sessions
-            violations.append('{}: scheduled scheduled in too small room'.format(self.group))
+            if include_fixed or not (self.is_fixed and o.is_fixed)
-            cost += self.business_constraint_costs['session_requires_trim']
+        )
 
-        if self.requested_duration > my_timeslot.duration:
+        if include_fixed or (not self.is_fixed):
-            violations.append('{}: scheduled scheduled in too short timeslot'.format(self.group))
+            if self.attendees > my_timeslot.capacity:
-            cost += self.business_constraint_costs['session_requires_trim']
+                violations.append('{}: scheduled in too small room'.format(self.group))
+                cost += self.business_constraint_costs['session_requires_trim']
 
-        if my_timeslot.time_group in self.timeranges_unavailable:
+            if self.requested_duration > my_timeslot.duration:
-            violations.append('{}: scheduled in unavailable timerange {}'
+                violations.append('{}: scheduled in too short timeslot'.format(self.group))
-                              .format(self.group, my_timeslot.time_group))
+                cost += self.business_constraint_costs['session_requires_trim']
-            cost += self.timeranges_unavailable_penalty
+
+            if my_timeslot.time_group in self.timeranges_unavailable:
+                violations.append('{}: scheduled in unavailable timerange {}'
+                                  .format(self.group, my_timeslot.time_group))
+                cost += self.timeranges_unavailable_penalty
             
         v, c = self._calculate_cost_overlapping_groups(overlapping_sessions)
         violations += v
@@ -669,7 +857,7 @@ class Session(object):
         violations += v
         cost += c
 
-        if self.wg_adjacent:
+        if self.wg_adjacent and (include_fixed or not self.is_fixed):
             adjacent_groups = tuple([schedule[t].group for t in my_timeslot.adjacent if t in schedule])
             if self.wg_adjacent not in adjacent_groups:
                 violations.append('{}: missing adjacency with {}, adjacents are: {}'
@@ -685,6 +873,8 @@ class Session(object):
         for other in overlapping_sessions:
             if not other:
                 continue
+            if self.is_fixed and other.is_fixed:
+                continue
             if other.group == self.group:
                 violations.append('{}: scheduled twice in overlapping slots'.format(self.group))
                 cost += math.inf
@@ -705,12 +895,14 @@ class Session(object):
         for other in overlapping_sessions:
             if not other:
                 continue
+            if self.is_fixed and other.is_fixed:
+                continue
             # BOFs cannot conflict with PRGs
             if self.is_bof and other.is_prg:
                 violations.append('{}: BOF overlaps with PRG: {}'
                                   .format(self.group, other.group))
                 cost += self.business_constraint_costs['bof_overlapping_prg']
-            # BOFs cannot conflict with any other BOFs 
+            # BOFs cannot conflict with any other BOFs
             if self.is_bof and other.is_bof:
                 violations.append('{}: BOF overlaps with other BOF: {}'
                                   .format(self.group, other.group))
@@ -720,7 +912,7 @@ class Session(object):
                 violations.append('{}: BOF overlaps with other session from same area: {}'
                                   .format(self.group, other.group))
                 cost += self.business_constraint_costs['bof_overlapping_area_wg']
-            # BOFs cannot conflict with any area-wide meetings (of any area) 
+            # BOFs cannot conflict with any area-wide meetings (of any area)
             if self.is_bof and other.is_area_meeting:
                 violations.append('{}: BOF overlaps with area meeting {}'
                                   .format(self.group, other.group))
@@ -743,23 +935,34 @@ class Session(object):
     
     @lru_cache(maxsize=10000)
     def _calculate_cost_my_other_sessions(self, my_sessions):
+        """Calculate cost due to other sessions for same group
+
+        my_sessions is a set of (TimeSlot, Session) tuples.
+        """
+        def sort_sessions(timeslot_session_pairs):
+            return sorted(timeslot_session_pairs, key=lambda item: item[1].session_pk)
+
         violations, cost = [], 0
-        my_sessions = list(my_sessions)
         if len(my_sessions) >= 2:
-            if my_sessions != sorted(my_sessions, key=lambda i: i[1].session_pk):
+            my_fixed_sessions = [m for m in my_sessions if m[1].is_fixed]
-                session_order = [s.session_pk for t, s in my_sessions]
+            fixed_sessions_in_order = (my_fixed_sessions == sort_sessions(my_fixed_sessions))
+            # Only possible to keep sessions in order if fixed sessions are in order - ignore cost if not.
+            if fixed_sessions_in_order and (list(my_sessions) != sort_sessions(my_sessions)):
+                session_order = [s.session_pk for t, s in list(my_sessions)]
                 violations.append('{}: sessions out of order: {}'.format(self.group, session_order))
                 cost += self.business_constraint_costs['sessions_out_of_order']
                 
-        if self.time_relation and len(my_sessions) >= 2:
+            if self.time_relation:
-            group_days = [t.start.date() for t, s in my_sessions]
+                group_days = [t.start.date() for t, s in my_sessions]
-            difference_days = abs((group_days[1] - group_days[0]).days)
+                # ignore conflict between two fixed sessions
-            if self.time_relation == 'subsequent-days' and difference_days != 1:
+                if not (my_sessions[0][1].is_fixed and my_sessions[1][1].is_fixed):
-                violations.append('{}: has time relation subsequent-days but difference is {}'
+                    difference_days = abs((group_days[1] - group_days[0]).days)
-                                  .format(self.group, difference_days))
+                    if self.time_relation == 'subsequent-days' and difference_days != 1:
-                cost += self.time_relation_penalty
+                        violations.append('{}: has time relation subsequent-days but difference is {}'
-            elif self.time_relation == 'one-day-seperation' and difference_days == 1:
+                                          .format(self.group, difference_days))
-                violations.append('{}: has time relation one-day-seperation but difference is {}'
+                        cost += self.time_relation_penalty
-                                  .format(self.group, difference_days))
+                    elif self.time_relation == 'one-day-seperation' and difference_days == 1:
-                cost += self.time_relation_penalty
+                        violations.append('{}: has time relation one-day-seperation but difference is {}'
+                                          .format(self.group, difference_days))
+                        cost += self.time_relation_penalty
         return violations, cost

ietf/meeting/models.py

@@ -758,6 +758,14 @@ class Schedule(models.Model):
     def qs_assignments_with_sessions(self):
         return self.assignments.filter(session__isnull=False)
 
+    def qs_timeslots_in_use(self):
+        """Get QuerySet containing timeslots used by the schedule"""
+        return TimeSlot.objects.filter(sessionassignments__schedule=self)
+
+    def qs_sessions_scheduled(self):
+        """Get QuerySet containing sessions assigned to timeslots by this schedule"""
+        return Session.objects.filter(timeslotassignments__schedule=self)
+
     def delete_schedule(self):
         self.assignments.all().delete()
         self.delete()

ietf/meeting/tests_schedule_generator.py

@@ -8,9 +8,12 @@ from django.core.management.base import CommandError
 from ietf.utils.test_utils import TestCase
 from ietf.group.factories import GroupFactory, RoleFactory
 from ietf.person.factories import PersonFactory
-from ietf.meeting.models import Constraint, TimerangeName, BusinessConstraint
+from ietf.meeting.models import Constraint, TimerangeName, BusinessConstraint, SchedTimeSessAssignment, Schedule
-from ietf.meeting.factories import MeetingFactory, RoomFactory, TimeSlotFactory, SessionFactory
+from ietf.meeting.factories import MeetingFactory, RoomFactory, TimeSlotFactory, SessionFactory, ScheduleFactory
-from ietf.meeting.management.commands.generate_schedule import ScheduleHandler
+from ietf.meeting.management.commands import generate_schedule
+from ietf.name.models import ConstraintName
+
+import debug                            # pyflakes:ignore
 
 
 class ScheduleGeneratorTest(TestCase):
@@ -58,16 +61,17 @@ class ScheduleGeneratorTest(TestCase):
 
         self.person1 = PersonFactory()
 
+        self.stdout = StringIO()
+
     def test_normal_schedule(self):
-        stdout = StringIO()
         self._create_basic_sessions()
-        generator = ScheduleHandler(stdout, self.meeting.number, verbosity=3)
+        generator = generate_schedule.ScheduleHandler(self.stdout, self.meeting.number, verbosity=3)
         violations, cost = generator.run()
         self.assertEqual(violations, self.fixed_violations)
         self.assertEqual(cost, self.fixed_cost)
 
-        stdout.seek(0)
+        self.stdout.seek(0)
-        output = stdout.read()
+        output = self.stdout.read()
         self.assertIn('WARNING: session wg2 (pk 13) has no attendees set', output)
         self.assertIn('scheduling 13 sessions in 20 timeslots', output)
         self.assertIn('Optimiser starting run 1', output)
@@ -77,7 +81,6 @@ class ScheduleGeneratorTest(TestCase):
         self.assertEqual(schedule.assignments.count(), 13)
 
     def test_unresolvable_schedule(self):
-        stdout = StringIO()
         self._create_basic_sessions()
         for group in self.all_groups:
             group.parent = self.area1
@@ -88,29 +91,170 @@ class ScheduleGeneratorTest(TestCase):
             Constraint.objects.create(meeting=self.meeting, source=group,
                                       name_id='bethere', person=self.person1)
 
-        generator = ScheduleHandler(stdout, self.meeting.number, verbosity=2)
+        generator = generate_schedule.ScheduleHandler(self.stdout, self.meeting.number, verbosity=2)
         violations, cost = generator.run()
         self.assertNotEqual(violations, [])
         self.assertGreater(cost, self.fixed_cost)
 
-        stdout.seek(0)
+        self.stdout.seek(0)
-        output = stdout.read()
+        output = self.stdout.read()
         self.assertIn('Optimiser did not find perfect schedule', output)
 
     def test_too_many_sessions(self):
-        stdout = StringIO()
         self._create_basic_sessions()
         self._create_basic_sessions()
         with self.assertRaises(CommandError):
-            generator = ScheduleHandler(stdout, self.meeting.number, verbosity=0)
+            generator = generate_schedule.ScheduleHandler(self.stdout, self.meeting.number, verbosity=0)
             generator.run()
 
     def test_invalid_meeting_number(self):
-        stdout = StringIO()
         with self.assertRaises(CommandError):
-            generator = ScheduleHandler(stdout, 'not-valid-meeting-number-aaaa', verbosity=0)
+            generator = generate_schedule.ScheduleHandler(self.stdout, 'not-valid-meeting-number-aaaa', verbosity=0)
             generator.run()
 
+    def test_base_schedule(self):
+        self._create_basic_sessions()
+        base_schedule = self._create_base_schedule()
+        assignment = base_schedule.assignments.first()
+        base_session = assignment.session
+        base_timeslot = assignment.timeslot
+
+        generator = generate_schedule.ScheduleHandler(
+            self.stdout,
+            self.meeting.number,
+            verbosity=3,
+            base_id=generate_schedule.ScheduleId.from_schedule(base_schedule),
+        )
+        violations, cost = generator.run()
+
+        expected_violations = self.fixed_violations + [
+            '{}: scheduled in too small room'.format(base_session.group.acronym),
+        ]
+        expected_cost = sum([
+            self.fixed_cost,
+            BusinessConstraint.objects.get(slug='session_requires_trim').penalty,
+        ])
+
+        self.assertEqual(violations, expected_violations)
+        self.assertEqual(cost, expected_cost)
+
+        generated_schedule = Schedule.objects.get(name=generator.name)
+        self.assertEqual(generated_schedule.base, base_schedule,
+                         'Base schedule should be attached to generated schedule')
+        self.assertCountEqual(
+            [a.session for a in base_timeslot.sessionassignments.all()],
+            [base_session],
+            'A session must not be scheduled on top of a base schedule assignment',
+        )
+
+        self.stdout.seek(0)
+        output = self.stdout.read()
+        self.assertIn('Applying schedule {} as base schedule'.format(
+            generate_schedule.ScheduleId.from_schedule(base_schedule)
+        ), output)
+        self.assertIn('WARNING: session wg2 (pk 13) has no attendees set', output)
+        self.assertIn('scheduling 13 sessions in 19 timeslots', output)  # 19 because base is using one
+        self.assertIn('Optimiser starting run 1', output)
+        self.assertIn('Optimiser found an optimal schedule', output)
+
+    def test_base_schedule_dynamic_cost(self):
+        """Conflicts with the base schedule should contribute to dynamic cost"""
+        # create the base schedule
+        base_schedule = self._create_base_schedule()
+        assignment = base_schedule.assignments.first()
+        base_session = assignment.session
+        base_timeslot = assignment.timeslot
+
+        # create another base session that conflicts with the first
+        SessionFactory(
+            meeting=self.meeting,
+            group=self.wg2,
+            attendees=10,
+            add_to_schedule=False,
+        )
+        SchedTimeSessAssignment.objects.create(
+            schedule=base_schedule,
+            session=SessionFactory(meeting=self.meeting, group=self.wg2, attendees=10, add_to_schedule=False),
+            timeslot=self.meeting.timeslot_set.filter(
+                time=base_timeslot.time + datetime.timedelta(days=1)
+            ).exclude(
+                sessionassignments__schedule=base_schedule
+            ).first(),
+        )
+        # make the base session group conflict with wg1 and wg2
+        Constraint.objects.create(
+            meeting=self.meeting,
+            source=base_session.group,
+            name_id='tech_overlap',
+            target=self.wg1,
+        )
+        Constraint.objects.create(
+            meeting=self.meeting,
+            source=base_session.group,
+            name_id='wg_adjacent',
+            target=self.wg2,
+        )
+
+        # create the session to schedule that will conflict
+        conflict_session = SessionFactory(meeting=self.meeting, group=self.wg1, add_to_schedule=False,
+                                          attendees=10, requested_duration=datetime.timedelta(hours=1))
+        conflict_timeslot = self.meeting.timeslot_set.filter(
+            time=base_timeslot.time,  # same time as base session
+            location__capacity__gte=conflict_session.attendees,  # no capacity violation
+        ).exclude(
+            sessionassignments__schedule=base_schedule  # do not use the same timeslot
+        ).first()
+
+        # Create the ScheduleHandler with the base schedule
+        handler = generate_schedule.ScheduleHandler(
+            self.stdout,
+            self.meeting.number,
+            max_cycles=1,
+            base_id=generate_schedule.ScheduleId.from_schedule(base_schedule),
+        )
+
+        # run once to be sure everything is primed, we'll ignore the outcome
+        handler.run()
+
+        timeslot_lut = {ts.timeslot_pk: ts for ts in handler.schedule.timeslots}
+        session_lut = {sess.session_pk: sess for sess in handler.schedule.sessions}
+        # now create schedule with a conflict
+        handler.schedule.schedule = {
+            timeslot_lut[conflict_timeslot.pk]: session_lut[conflict_session.pk],
+        }
+
+        # check that we get the expected dynamic cost - should NOT include conflict with wg2
+        # because that is in the base schedule
+        violations, cost = handler.schedule.calculate_dynamic_cost()
+        self.assertCountEqual(
+            violations,
+            ['{}: group conflict with {}'.format(base_session.group.acronym, self.wg1.acronym)]
+        )
+        self.assertEqual(
+            cost,
+            ConstraintName.objects.get(pk='tech_overlap').penalty,
+        )
+
+        # check the total cost - now should see wg2 and capacity conflicts
+        violations, cost = handler.schedule.total_schedule_cost()
+        self.assertCountEqual(
+            violations,
+            [
+                '{}: group conflict with {}'.format(base_session.group.acronym, self.wg1.acronym),
+                '{}: missing adjacency with {}, adjacents are: '.format(base_session.group.acronym, self.wg2.acronym),
+                '{}: scheduled in too small room'.format(base_session.group.acronym),
+            ]
+        )
+        self.assertEqual(
+            cost,
+            sum([
+                BusinessConstraint.objects.get(pk='session_requires_trim').penalty,
+                ConstraintName.objects.get(pk='wg_adjacent').penalty,
+                ConstraintName.objects.get(pk='tech_overlap').penalty,
+            ]),
+        )
+
+
     def _create_basic_sessions(self):
         for group in self.all_groups:
             SessionFactory(meeting=self.meeting, group=group, add_to_schedule=False, attendees=5,
@@ -146,3 +290,31 @@ class ScheduleGeneratorTest(TestCase):
                                  'No timeslot with sufficient capacity available for wg2, '
                                  'requested 500, trimmed to 100']
         self.fixed_cost = BusinessConstraint.objects.get(slug='session_requires_trim').penalty * 2
+
+    def _create_base_schedule(self):
+        """Create a base schedule
+
+        Generates a base schedule using the first Monday timeslot with a location
+        with capacity smaller than 200.
+        """
+        base_schedule = ScheduleFactory(meeting=self.meeting)
+        base_reg_session = SessionFactory(
+            meeting=self.meeting,
+            requested_duration=datetime.timedelta(minutes=60),
+            attendees=200,
+           add_to_schedule=False
+        )
+        # use a timeslot not on Sunday
+        ts = self.meeting.timeslot_set.filter(
+            time__gt=self.meeting.date + datetime.timedelta(days=1),
+            location__capacity__lt=base_reg_session.attendees,
+        ).order_by(
+            'time'
+        ).first()
+        SchedTimeSessAssignment.objects.create(
+            schedule=base_schedule,
+            session=base_reg_session,
+            timeslot=ts,
+        )
+        return base_schedule
+