From 4cd3c484be6e162a8b70a7cf1ab29b78fc63305e Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Fri, 14 Oct 2016 15:03:13 +0000 Subject: [PATCH] New management command create_group_wikis. Work in progress. - Legacy-Id: 12149 --- ietf/settings.py | 32 +- .../management/commands/create_group_wikis.py | 281 ++++++++++++++++++ 2 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 ietf/utils/management/commands/create_group_wikis.py diff --git a/ietf/settings.py b/ietf/settings.py index 54a7ec1f6..2ff9b2cef 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -659,7 +659,7 @@ USER_PREFERENCE_DEFAULTS = { "left_menu" : "on", } -TRAC_ADMIN_CMD = "/usr/bin/trac-admin" +TRAC_MASTER_DIR = "/a/www/trac-setup/" TRAC_WIKI_DIR_ROOT = "/a/www/www6s/trac/" TRAC_WIKI_DIR_PATTERN = os.path.join(TRAC_WIKI_DIR_ROOT, "%s") TRAC_WIKI_URL_PATTERN = "https://trac.ietf.org/trac/%s/wiki" @@ -667,6 +667,36 @@ TRAC_ISSUE_URL_PATTERN = "https://trac.ietf.org/trac/%s/report/1" TRAC_SVN_DIR_PATTERN = "/a/svn/group/%s" TRAC_SVN_URL_PATTERN = "https://svn.ietf.org/svn/group/%s/" +TRAC_ENV_OPTIONS = [ + ('project', 'name', "{name} Wiki"), + ('trac', 'database', 'sqlite:db/trac.db' ), + ('trac', 'repository_type', 'svn'), + ('trac', 'repository_dir', "{svn_dir}"), + ('inherit', 'file', "/a/www/trac-setup/conf/trac.ini"), +] + +TRAC_WIKI_PAGES_TEMPLATES = [ + "utils/wiki/IetfSpecificFeatures", + "utils/wiki/InterMapTxt", + "utils/wiki/SvnTracHooks", + "utils/wiki/ThisTracInstallation", + "utils/wiki/TrainingMaterials", + "utils/wiki/WikiStart", +] + +TRAC_ISSUE_SEVERITY_ADD = [ + "-", + "Candidate WG Document", + "Active WG Document", + "Waiting for Expert Review", + "In WG Last Call", + "Waiting for Shepherd Writeup", + "Submitted WG Document", + "Dead WG Document", +] + +SVN_ADMIN_COMMAND = "/usr/bin/svnadmin" + # Email addresses people attempt to set for their account will be checked # against the following list of regex expressions with re.search(pat, addr): EXLUDED_PERSONAL_EMAIL_REGEX_PATTERNS = ["@ietf.org$"] diff --git a/ietf/utils/management/commands/create_group_wikis.py b/ietf/utils/management/commands/create_group_wikis.py new file mode 100644 index 000000000..897539da3 --- /dev/null +++ b/ietf/utils/management/commands/create_group_wikis.py @@ -0,0 +1,281 @@ +# Copyright 2016 IETF Trust + +import os +import copy +import syslog +import pkg_resources +#from optparse import make_option + +from trac.admin.api import AdminCommandManager +from trac.core import TracError +from trac.env import Environment +from trac.perm import PermissionSystem +from trac.ticket.model import Component, Milestone, Severity +from trac.util.text import unicode_unquote +from trac.versioncontrol.api import RepositoryManager +from trac.wiki.admin import WikiAdmin +from trac.wiki.model import WikiPage + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.template.loader import render_to_string + +import debug # pyflakes:ignore + +from ietf.group.models import Group, GroupURL +from ietf.utils.pipe import pipe + +logtag = __name__.split('.')[-1] +logname = "user.log" +syslog.openlog(logname, syslog.LOG_PID, syslog.LOG_USER) + +class Command(BaseCommand): + help = "Create group wikis for WGs, RGs and Areas which don't have one." + + option_list = BaseCommand.option_list + ( + ) + #verbosity = 1 + + def note(self, msg): + if self.verbosity > 1: + self.stdout.write(msg) + + def log(self, msg): + syslog.syslog(msg) + self.stderr.write(msg) + + # --- svn --- + + def do_cmd(self, cmd, *args): + quoted_args = [ '"%s"'%a if ' ' in a else a for a in args ] + self.note("Running %s %s ..." % (os.path.basename(cmd), " ".join(quoted_args))) + command = [ cmd, ] + list(args) + code, out, err = pipe(command) + msg = None + if code != 0: + msg = "Error %s: %s when executing '%s'" % (code, err, " ".join(command)) + self.log(msg) + return msg, out + + def svn_admin_cmd(self, *args): + return self.do_cmd(settings.SVN_ADMIN_COMMAND, *args) + + def create_svn(self, svn): + self.note(" Creating svn repository: %s" % svn) + if not os.path.exists(os.path.dirname(svn)): + msg = "Intended to create '%s', but parent directory is missing" % svn + self.log(msg) + return msg + err, out= self.svn_admin_cmd("create", svn ) + return err + + # --- trac --- + + def remove_demo_components(self, group, env): + for component in Component.select(env): + if component.name.startswith('component'): + component.delete() + + def remove_demo_milestones(self, group, env): + for milestone in Milestone.select(env): + if milestone.name.startswith('milestone'): + milestone.delete() + + def symlink_to_master_assets(self, group, env): + master_dir = settings.TRAC_MASTER_DIR + master_htdocs = os.path.join(master_dir, "htdocs") + group_htdocs = os.path.join(group.trac_dir, "htdocs") + self.note(" Symlinking %s to %s" % (master_htdocs, group_htdocs)) + os.removedirs(group_htdocs) + os.symlink(master_htdocs, group_htdocs) + + def add_wg_draft_states(self, group, env): + for state in settings.TRAC_ISSUE_SEVERITY_ADD: + self.note(" Adding severity %s" % state) + severity = Severity(env) + severity.name = state + severity.insert() + + def add_wiki_page(self, env, name, text): + page = WikiPage(env, name) + if page.time: + self.note(" ** Page %s already exists, not adding it." % name) + return + page.text = text + page.save(author="(System)", comment="Initial page import") + + def add_default_wiki_pages(self, group, env): + dir = pkg_resources.resource_filename('trac.wiki', 'default-pages') + #WikiAdmin(env).load_pages(dir) + with env.db_transaction: + for name in os.listdir(dir): + filename = os.path.join(dir, name) + name = unicode_unquote(name.encode('utf-8')) + if os.path.isfile(filename): + self.note(" Adding page %s" % name) + with open(filename) as file: + text = file.read().decode('utf-8') + self.add_wiki_page(env, name, text) + + def add_custom_wiki_pages(self, group, env): + for templ in settings.TRAC_WIKI_PAGES_TEMPLATES: + _, name = os.path.split(templ) + text = render_to_string(templ, {"group": group}) + self.note(" Adding page %s" % name) + + def sync_default_repository(self, group, env): + repository = env.get_repository('') + if repository: + self.note(" Indexing default repository") + repository.sync() + + def create_trac(self, group): + if not os.path.exists(os.path.dirname(group.trac_dir)): + msg = "Intended to create '%s', but parent directory is missing" % group.trac_dir + self.log(msg) + return None + options = copy.deepcopy(settings.TRAC_ENV_OPTIONS) + # Interpolate group field names to values in the option settings: + for i in range(len(options)): + sect, key, val = options[i] + val = val.format(**group.__dict__) + options[i] = sect, key, val + # Try to creat ethe environment, remove unwanted defaults, and add + # custom pages and settings. + try: + debug.show('options') + env = Environment(group.trac_dir, create=True, options=options) + self.remove_demo_components(group, env) + self.remove_demo_milestones(group, env) + self.maybe_add_group_url(group, 'Wiki', settings.TRAC_WIKI_URL_PATTERN % group.acronym) + self.maybe_add_group_url(group, 'Issue tracker', settings.TRAC_ISSUE_URL_PATTERN % group.acronym) + # Use custom assets (if any) from the master setup + self.symlink_to_master_assets(group, env) + if group.type_id == 'wg': + self.add_wg_draft_states(group, env) + self.add_default_wiki_pages(group, env) + self.add_custom_wiki_pages(group, env) + self.sync_default_repository(group, env) + # Components (i.e., drafts) will be handled during components + # update later + # Permissions will be handled during permission update later. + return env + except TracError as e: + self.log("While creating trac instance for %s: %s" % (group, e)) + raise + return None + + def update_trac_permissions(self, group, env): + mgr = PermissionSystem(env) + permission_list = mgr.get_all_permissions() + permission_list = [ (u,a) for (u,a) in permission_list if not u in ['anonymous', 'authenticated']] + permissions = {} + for user, action in permission_list: + if not user in permissions: + permissions[user] = [] + permissions[user].append(action) + roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) + users = [] + for role in roles: + user = role.email.address.lower() + users.append(user) + if not user in permissions: + try: + mgr.grant_permission(user, 'TRAC_ADMIN') + self.note(" Granting admin permission for %s" % user) + except TracError as e: + self.log("While adding admin permission for %s: %s" (user, e)) + for user in permissions: + if not user in users: + if 'TRAC_ADMIN' in permissions[user]: + try: + self.note(" Revoking admin permission for %s" % user) + mgr.revoke_permission(user, 'TRAC_ADMIN') + except TracError as e: + self.log("While revoking admin permission for %s: %s" (user, e)) + + def update_trac_components(self, group, env): + components = Component.select(env) + comp_names = [ c.name for c in components ] + group_docs = group.document_set.filter(states__slug='active', type_id='draft').distinct() + group_comp = [] + for doc in group_docs: + if not doc.name.startswith('draft-'): + self.log("While adding components: unexpectd %s group doc name: %s" % (group.acronym, doc.name)) + continue + name = doc.name[len('draft-'):] + if name.startswith('ietf-'): + name = name[len('ietf-'):] + elif name.startswith('irtf-'): + name = name[len('ietf-'):] + if name.startswith(group.acronym+'-'): + name = name[len(group.acronym+'-'):] + group_comp.append(name) + if not name in comp_names and not doc.name in comp_names: + self.note(" Group draft: %s" % doc.name) + self.note(" Adding component %s" % name) + comp = Component(env) + comp.name = name + comp.owner = "%s@ietf.org" % doc.name + comp.insert() + + def maybe_add_group_url(self, group, name, url): + urls = [ u for u in group.groupurl_set.all() if name.lower() in u.name.lower() ] + if not urls: + self.note(" adding %s %s URL ..." % (group.acronym, name.lower())) + group.groupurl_set.add(GroupURL(group=group, name=name, url=url)) + + def add_custom_pages(self, group, env): + for template_name in settings.TRAC_WIKI_PAGES_TEMPLATES: + pass + + def add_custom_group_states(self, group, env): + for state_name in settings.TRAC_ISSUE_SEVERITY_ADD: + pass + + # -------------------------------------------------------------------- + + def handle(self, *filenames, **options): + self.verbosity = options['verbosity'] + self.errors = 0 + + if self.verbosity.isdigit(): + self.verbosity = int(self.verbosity) + + if not os.path.exists(settings.TRAC_WIKI_DIR_ROOT): + raise CommandError('The Wiki base direcory specified in settings.TRAC_WIKI_DIR_ROOT (%s) does not exist.' % settings.TRAC_WIKI_DIR_ROOT) + + groups = Group.objects.filter( + type__slug__in=['wg','rg','area'], + state__slug='active' + ) + + for group in groups: + try: + self.note("Processing group %s" % group.acronym) + group.trac_dir = settings.TRAC_WIKI_DIR_PATTERN % group.acronym + group.svn_dir = settings.TRAC_SVN_DIR_PATTERN % group.acronym + + if not os.path.exists(group.svn_dir): + err = self.create_svn(group.svn_dir) + self.errors += 1 if err else 0 + + if not os.path.exists(group.trac_dir): + trac_env = self.create_trac(group) + self.errors += 1 if not trac_env else 0 + else: + trac_env = Environment(group.trac_dir) + + if not trac_env: + continue + + self.update_trac_permissions(group, trac_env) + self.update_trac_components(group, trac_env) + + except Exception as e: + self.errors += 1 + self.log("While processing %s: %s" % (group.acronym, e)) + raise + + if self.errors: + raise CommandError("There were %s failures in WG Trac creation, see syslog %s for details." % (self.errors, logname))