340 lines
14 KiB
Python
340 lines
14 KiB
Python
'''
|
|
proc_utils.py
|
|
|
|
This module contains all the functions for generating static proceedings pages
|
|
'''
|
|
import datetime
|
|
import httplib2
|
|
import os
|
|
import re
|
|
import subprocess
|
|
from urllib import urlencode
|
|
|
|
import debug # pyflakes:ignore
|
|
|
|
from apiclient.discovery import build
|
|
from django.conf import settings
|
|
from django.core.exceptions import ObjectDoesNotExist
|
|
|
|
from ietf.doc.models import Document, DocAlias, DocEvent, NewRevisionDocEvent, State
|
|
from ietf.group.models import Group
|
|
from ietf.meeting.models import Meeting, SessionPresentation, TimeSlot, SchedTimeSessAssignment
|
|
from ietf.person.models import Person
|
|
from ietf.utils.log import log
|
|
from ietf.utils.mail import send_mail
|
|
|
|
AUDIO_FILE_RE = re.compile(r'ietf(?P<number>[\d]+)-(?P<room>.*)-(?P<time>[\d]{8}-[\d]{4})')
|
|
VIDEO_TITLE_RE = re.compile(r'IETF(?P<number>[\d]+)-(?P<name>.*)-(?P<date>\d{8})-(?P<time>\d{4})')
|
|
|
|
|
|
|
|
def import_youtube_video_urls(meeting, http=httplib2.Http()):
|
|
'''Create Document and set external_url for session videos'''
|
|
youtube = build(settings.YOUTUBE_API_SERVICE_NAME, settings.YOUTUBE_API_VERSION,
|
|
developerKey=settings.YOUTUBE_API_KEY, http=http)
|
|
playlistid = get_youtube_playlistid(youtube, 'IETF' + meeting.number)
|
|
if playlistid is None:
|
|
return None
|
|
for video in get_youtube_videos(youtube, playlistid):
|
|
match = VIDEO_TITLE_RE.match(video['title'])
|
|
if match:
|
|
session = _get_session(**match.groupdict())
|
|
if session:
|
|
url = video['url']
|
|
get_or_create_recording_document(url,session)
|
|
|
|
def get_youtube_playlistid(youtube, title, http=httplib2.Http()):
|
|
'''Returns the youtube playlistId matching title string, a string'''
|
|
request = youtube.search().list(
|
|
q=title,
|
|
part='id,snippet',
|
|
channelId=settings.YOUTUBE_IETF_CHANNEL_ID,
|
|
type='playlist',
|
|
maxResults=1
|
|
)
|
|
search_response = request.execute(http=http)
|
|
|
|
try:
|
|
playlistid = search_response['items'][0]['id']['playlistId']
|
|
except (KeyError, IndexError):
|
|
return None
|
|
return playlistid
|
|
|
|
def get_youtube_videos(youtube, playlistid, http=httplib2.Http()):
|
|
'''Returns list of dictionaries with title, urls keys'''
|
|
videos = []
|
|
kwargs = dict(part="snippet",playlistId=playlistid,maxResults=50)
|
|
playlistitems = youtube.playlistItems()
|
|
request = playlistitems.list(**kwargs)
|
|
# handle pagination
|
|
while request is not None:
|
|
playlistitems_doc = request.execute(http=http)
|
|
videos.extend(_get_urls_from_json(playlistitems_doc))
|
|
request = playlistitems.list_next(request, playlistitems_doc)
|
|
return videos
|
|
|
|
def _get_session(number,name,date,time):
|
|
'''Lookup session using data from video title'''
|
|
meeting = Meeting.objects.get(number=number)
|
|
schedule = meeting.agenda
|
|
timeslot_time = datetime.datetime.strptime(date + time,'%Y%m%d%H%M')
|
|
try:
|
|
assignment = SchedTimeSessAssignment.objects.get(
|
|
schedule = schedule,
|
|
session__group__acronym = name.lower(),
|
|
timeslot__time = timeslot_time,
|
|
)
|
|
except (SchedTimeSessAssignment.DoesNotExist, SchedTimeSessAssignment.MultipleObjectsReturned):
|
|
return None
|
|
|
|
return assignment.session
|
|
|
|
def _get_urls_from_json(doc):
|
|
'''Returns list of dictonary titel,url from search results'''
|
|
urls = []
|
|
for item in doc['items']:
|
|
title = item['snippet']['title']
|
|
#params = dict(v=item['snippet']['resourceId']['videoId'], list=item['snippet']['playlistId'])
|
|
params = [('v',item['snippet']['resourceId']['videoId']), ('list',item['snippet']['playlistId'])]
|
|
url = settings.YOUTUBE_BASE_URL + '?' + urlencode(params)
|
|
urls.append(dict(title=title, url=url))
|
|
return urls
|
|
|
|
def import_audio_files(meeting):
|
|
'''
|
|
Checks for audio files and creates corresponding materials (docs) for the Session
|
|
Expects audio files in the format ietf[meeting num]-[room]-YYYMMDD-HHMM.*,
|
|
|
|
Example: ietf90-salonb-20140721-1710.mp3
|
|
'''
|
|
unmatched_files = []
|
|
path = os.path.join(settings.MEETING_RECORDINGS_DIR, meeting.type.slug + meeting.number)
|
|
if not os.path.exists(path):
|
|
return None
|
|
for filename in os.listdir(path):
|
|
timeslot = get_timeslot_for_filename(filename)
|
|
if timeslot:
|
|
sessionassignments = timeslot.sessionassignments.filter(
|
|
schedule=timeslot.meeting.agenda,
|
|
session__status='sched',
|
|
).exclude(session__agenda_note__icontains='canceled').order_by('timeslot__time')
|
|
if not sessionassignments:
|
|
continue
|
|
url = settings.IETF_AUDIO_URL + 'ietf{}/{}'.format(meeting.number, filename)
|
|
doc = get_or_create_recording_document(url,sessionassignments[0].session)
|
|
attach_recording(doc, [ x.session for x in sessionassignments ])
|
|
else:
|
|
# use for reconciliation email
|
|
unmatched_files.append(filename)
|
|
|
|
if unmatched_files:
|
|
send_audio_import_warning(unmatched_files)
|
|
|
|
|
|
def get_timeslot_for_filename(filename):
|
|
'''Returns a timeslot matching the filename given.
|
|
NOTE: currently only works with ietfNN prefix (regular meetings)
|
|
'''
|
|
basename, _ = os.path.splitext(filename)
|
|
match = AUDIO_FILE_RE.match(basename)
|
|
if match:
|
|
try:
|
|
meeting = Meeting.objects.get(number=match.groupdict()['number'])
|
|
room_mapping = {normalize_room_name(room.name): room.name for room in meeting.room_set.all()}
|
|
time = datetime.datetime.strptime(match.groupdict()['time'],'%Y%m%d-%H%M')
|
|
slots = TimeSlot.objects.filter(
|
|
meeting=meeting,
|
|
location__name=room_mapping[match.groupdict()['room']],
|
|
time=time,
|
|
sessionassignments__schedule=meeting.agenda,
|
|
).distinct()
|
|
return slots.get()
|
|
except (ObjectDoesNotExist, KeyError):
|
|
return None
|
|
|
|
def attach_recording(doc, sessions):
|
|
'''Associate recording document with sessions'''
|
|
for session in sessions:
|
|
if doc not in session.materials.all():
|
|
# add document to session
|
|
presentation = SessionPresentation.objects.create(
|
|
session=session,
|
|
document=doc,
|
|
rev=doc.rev)
|
|
session.sessionpresentation_set.add(presentation)
|
|
if not doc.docalias_set.filter(name__startswith='recording-{}-{}'.format(session.meeting.number,session.group.acronym)):
|
|
sequence = get_next_sequence(session.group,session.meeting,'recording')
|
|
name = 'recording-{}-{}-{}'.format(session.meeting.number,session.group.acronym,sequence)
|
|
doc.docalias_set.create(name=name)
|
|
|
|
def normalize_room_name(name):
|
|
'''Returns room name converted to be used as portion of filename'''
|
|
return name.lower().replace(' ','').replace('/','_')
|
|
|
|
def get_or_create_recording_document(url,session):
|
|
try:
|
|
return Document.objects.get(external_url=url)
|
|
except ObjectDoesNotExist:
|
|
return create_recording(session,url)
|
|
|
|
def create_recording(session, url, title=None, user=None):
|
|
'''
|
|
Creates the Document type=recording, setting external_url and creating
|
|
NewRevisionDocEvent
|
|
'''
|
|
sequence = get_next_sequence(session.group,session.meeting,'recording')
|
|
name = 'recording-{}-{}-{}'.format(session.meeting.number,session.group.acronym,sequence)
|
|
time = session.official_timeslotassignment().timeslot.time.strftime('%Y-%m-%d %H:%M')
|
|
if not title:
|
|
if url.endswith('mp3'):
|
|
title = 'Audio recording for {}'.format(time)
|
|
else:
|
|
title = 'Video recording for {}'.format(time)
|
|
|
|
doc = Document.objects.create(name=name,
|
|
title=title,
|
|
external_url=url,
|
|
group=session.group,
|
|
rev='00',
|
|
type_id='recording')
|
|
doc.set_state(State.objects.get(type='recording', slug='active'))
|
|
|
|
doc.docalias_set.create(name=name)
|
|
|
|
# create DocEvent
|
|
NewRevisionDocEvent.objects.create(type='new_revision',
|
|
by=user or Person.objects.get(name='(System)'),
|
|
doc=doc,
|
|
rev=doc.rev,
|
|
desc='New revision available',
|
|
time=doc.time)
|
|
pres = SessionPresentation.objects.create(session=session,document=doc,rev=doc.rev)
|
|
session.sessionpresentation_set.add(pres)
|
|
|
|
return doc
|
|
|
|
def get_next_sequence(group,meeting,type):
|
|
'''
|
|
Returns the next sequence number to use for a document of type = type.
|
|
Takes a group=Group object, meeting=Meeting object, type = string
|
|
'''
|
|
aliases = DocAlias.objects.filter(name__startswith='{}-{}-{}-'.format(type,meeting.number,group.acronym))
|
|
if not aliases:
|
|
return 1
|
|
aliases = aliases.order_by('name')
|
|
sequence = int(aliases.last().name.split('-')[-1]) + 1
|
|
return sequence
|
|
|
|
def send_audio_import_warning(unmatched_files):
|
|
'''Send email to interested parties that some audio files weren't matched to timeslots'''
|
|
send_mail(request = None,
|
|
to = settings.AUDIO_IMPORT_EMAIL,
|
|
frm = "IETF Secretariat <ietf-secretariat@ietf.org>",
|
|
subject = "Audio file import warning",
|
|
template = "proceedings/audio_import_warning.txt",
|
|
context = dict(unmatched_files=unmatched_files),
|
|
extra = {})
|
|
|
|
# -------------------------------------------------
|
|
# End Recording Functions
|
|
# -------------------------------------------------
|
|
|
|
def get_progress_stats(sdate,edate):
|
|
'''
|
|
This function takes a date range and produces a dictionary of statistics / objects for
|
|
use in a progress report. Generally the end date will be the date of the last meeting
|
|
and the start date will be the date of the meeting before that.
|
|
'''
|
|
data = {}
|
|
data['sdate'] = sdate
|
|
data['edate'] = edate
|
|
|
|
events = DocEvent.objects.filter(doc__type='draft',time__gte=sdate,time__lt=edate)
|
|
|
|
data['actions_count'] = events.filter(type='iesg_approved').count()
|
|
data['last_calls_count'] = events.filter(type='sent_last_call').count()
|
|
new_draft_events = events.filter(newrevisiondocevent__rev='00')
|
|
new_drafts = list(set([ e.doc_id for e in new_draft_events ]))
|
|
data['new_drafts_count'] = len(new_drafts)
|
|
data['new_drafts_updated_count'] = events.filter(doc__in=new_drafts,newrevisiondocevent__rev='01').count()
|
|
data['new_drafts_updated_more_count'] = events.filter(doc__in=new_drafts,newrevisiondocevent__rev='02').count()
|
|
|
|
update_events = events.filter(type='new_revision').exclude(doc__in=new_drafts)
|
|
data['updated_drafts_count'] = len(set([ e.doc_id for e in update_events ]))
|
|
|
|
# Calculate Final Four Weeks stats (ffw)
|
|
ffwdate = edate - datetime.timedelta(days=28)
|
|
ffw_new_count = events.filter(time__gte=ffwdate,newrevisiondocevent__rev='00').count()
|
|
try:
|
|
ffw_new_percent = format(ffw_new_count / float(data['new_drafts_count']),'.0%')
|
|
except ZeroDivisionError:
|
|
ffw_new_percent = 0
|
|
|
|
data['ffw_new_count'] = ffw_new_count
|
|
data['ffw_new_percent'] = ffw_new_percent
|
|
|
|
ffw_update_events = events.filter(time__gte=ffwdate,type='new_revision').exclude(doc__in=new_drafts)
|
|
ffw_update_count = len(set([ e.doc_id for e in ffw_update_events ]))
|
|
try:
|
|
ffw_update_percent = format(ffw_update_count / float(data['updated_drafts_count']),'.0%')
|
|
except ZeroDivisionError:
|
|
ffw_update_percent = 0
|
|
|
|
data['ffw_update_count'] = ffw_update_count
|
|
data['ffw_update_percent'] = ffw_update_percent
|
|
|
|
rfcs = events.filter(type='published_rfc')
|
|
data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level')
|
|
|
|
data['counts'] = {'std':rfcs.filter(doc__intended_std_level__in=('ps','ds','std')).count(),
|
|
'bcp':rfcs.filter(doc__intended_std_level='bcp').count(),
|
|
'exp':rfcs.filter(doc__intended_std_level='exp').count(),
|
|
'inf':rfcs.filter(doc__intended_std_level='inf').count()}
|
|
|
|
data['new_groups'] = Group.objects.filter(
|
|
type='wg',
|
|
groupevent__changestategroupevent__state='active',
|
|
groupevent__time__gte=sdate,
|
|
groupevent__time__lt=edate)
|
|
|
|
data['concluded_groups'] = Group.objects.filter(
|
|
type='wg',
|
|
groupevent__changestategroupevent__state='conclude',
|
|
groupevent__time__gte=sdate,
|
|
groupevent__time__lt=edate)
|
|
|
|
return data
|
|
|
|
def is_powerpoint(doc):
|
|
'''
|
|
Returns true if document is a Powerpoint presentation
|
|
'''
|
|
return doc.file_extension() in ('ppt','pptx')
|
|
|
|
def post_process(doc):
|
|
'''
|
|
Does post processing on uploaded file.
|
|
- Convert PPT to PDF
|
|
'''
|
|
if is_powerpoint(doc) and hasattr(settings,'SECR_PPT2PDF_COMMAND'):
|
|
try:
|
|
cmd = settings.SECR_PPT2PDF_COMMAND
|
|
cmd.append(doc.get_file_path()) # outdir
|
|
cmd.append(os.path.join(doc.get_file_path(),doc.external_url)) # filename
|
|
subprocess.check_call(cmd)
|
|
except (subprocess.CalledProcessError, OSError) as error:
|
|
log("Error converting PPT: %s" % (error))
|
|
return
|
|
# change extension
|
|
base,ext = os.path.splitext(doc.external_url)
|
|
doc.external_url = base + '.pdf'
|
|
|
|
e = DocEvent.objects.create(
|
|
type='changed_document',
|
|
by=Person.objects.get(name="(System)"),
|
|
doc=doc,
|
|
rev=doc.rev,
|
|
desc='Converted document to PDF',
|
|
)
|
|
doc.save_with_history([e])
|