diff --git a/env/.gitignore b/env/.gitignore new file mode 100644 index 000000000..e340628a0 --- /dev/null +++ b/env/.gitignore @@ -0,0 +1,7 @@ +/bin +/share +/selenium +/etc +/local +/lib +/include diff --git a/env/pip.conf b/env/pip.conf new file mode 100644 index 000000000..212621844 --- /dev/null +++ b/env/pip.conf @@ -0,0 +1,3 @@ +[install] +ignore-installed = true + diff --git a/ietf/checks.py b/ietf/checks.py index 07fb84068..e440d5038 100644 --- a/ietf/checks.py +++ b/ietf/checks.py @@ -1,5 +1,6 @@ import os import time +from textwrap import dedent from django.conf import settings from django.core import checks @@ -209,3 +210,32 @@ def check_cache(app_configs, **kwargs): errors.append(cache_error("Cache didn't accept session cookie age", "E0016")) return errors +@checks.register('cache') +def check_svn_import(app_configs, **kwargs): + errors = [] + if settings.SERVER_MODE == 'production': + try: + import svn # pyflakes:ignore + except ImportError as e: + errors.append(checks.Critical( + "Could not import the python svn module:\n %s\n" % e, + hint = dedent(""" + You are running in production mode, and the subversion bindings for python + are necessary in order to run the Trac wiki glue scripts. + + However, the subversion bindings seem to be unavailable. The subversion + bindings are not available for install using pip, but must be supplied by + the system package manager. In order to be available within a python + virtualenv, the virtualenv must have been created with the + --system-site-packages flag, so that the packages installed by the system + package manager are visible. + + Please install 'python-subversion' (Debian), 'subversion-python' (RedHat, + CentOS, Fedora), 'subversion-python27bindings' (BSD); and set up a + virtualenv using the --system-site-packages flag. Further tips are + available at https://trac.edgewall.org/wiki/TracSubversion. + + """).replace('\n', '\n ').rstrip(), + id = "datatracker.E0014", + )) + return errors diff --git a/ietf/settings.py b/ietf/settings.py index 25f9d8696..381afb9c9 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -382,7 +382,7 @@ TEST_URL_COVERAGE_EXCLUDE = [ r"^\^admin/", ] -# Tese are filename globs +# These are filename globs. They are fed directly to the coverage code checker. TEST_CODE_COVERAGE_EXCLUDE = [ "*/tests*", "*/admin.py", @@ -660,14 +660,43 @@ USER_PREFERENCE_DEFAULTS = { "left_menu" : "on", } -TRAC_ADMIN_CMD = "/usr/bin/trac-admin" -TRAC_WIKI_DIR_ROOT = "/a/www/www6s/trac/" -TRAC_WIKI_DIR_PATTERN = os.path.join(TRAC_WIKI_DIR_ROOT, "%s") +TRAC_MASTER_DIR = "/a/www/trac-setup/" +TRAC_WIKI_DIR_PATTERN = "/a/www/www6s/trac/%s" TRAC_WIKI_URL_PATTERN = "https://trac.ietf.org/trac/%s/wiki" TRAC_ISSUE_URL_PATTERN = "https://trac.ietf.org/trac/%s/report/1" TRAC_SVN_DIR_PATTERN = "/a/svn/group/%s" TRAC_SVN_URL_PATTERN = "https://svn.ietf.org/svn/group/%s/" +TRAC_ENV_OPTIONS = [ + ('project', 'name', "{name} Wiki"), + ('trac', 'database', 'sqlite:db/trac.db' ), + ('trac', 'repository_type', 'svn'), + ('trac', 'repository_dir', "{svn_dir}"), + ('inherit', 'file', "/a/www/trac-setup/conf/trac.ini"), +] + +TRAC_WIKI_PAGES_TEMPLATES = [ + "utils/wiki/IetfSpecificFeatures", + "utils/wiki/InterMapTxt", + "utils/wiki/SvnTracHooks", + "utils/wiki/ThisTracInstallation", + "utils/wiki/TrainingMaterials", + "utils/wiki/WikiStart", +] + +TRAC_ISSUE_SEVERITY_ADD = [ + "-", + "Candidate WG Document", + "Active WG Document", + "Waiting for Expert Review", + "In WG Last Call", + "Waiting for Shepherd Writeup", + "Submitted WG Document", + "Dead WG Document", +] + +SVN_ADMIN_COMMAND = "/usr/bin/svnadmin" + # Email addresses people attempt to set for their account will be checked # against the following list of regex expressions with re.search(pat, addr): EXLUDED_PERSONAL_EMAIL_REGEX_PATTERNS = ["@ietf.org$"] diff --git a/ietf/templates/utils/wiki/IetfSpecificFeatures b/ietf/templates/utils/wiki/IetfSpecificFeatures new file mode 100644 index 000000000..62502bc84 --- /dev/null +++ b/ietf/templates/utils/wiki/IetfSpecificFeatures @@ -0,0 +1,94 @@ += IETF-Specific Information = + +== Editing the Wiki and Issues == + +In order to create and edit wiki pages and issues, you need to log in. Click on the +small 'Login' link above the main horizontal menubar. You log in with the same +username (your email address) and password as for all other ''tools.ietf.org'' password +protected accesses. If you don't have a login/passwd or need to reset your passwd, go +to http://tools.ietf.org/newpasswd . + +The login and password is also used for commits to the SVN repository. See more about +the repository further down. + +== IETF-Specific Features == + +This Trac installation has a few IETF-specific features which are not generally found +in Trac: + + * Occurences of RFC numbers or draft names in Wiki text will generate links to the + RFC or draft in question. Unless you want to point to an RFC or draft in a + specific location which is different from the automatically generated link, you + don't need to explicitly add links for RFCs and drafts. Examples: RFC 2026, + draft-ietf-poised95-std-proc-3 + + * Each issue in the issue tracker can be indicated to concern a 'component'. This is + a standard Trac feature; however, the list of available components is automatically + updated to include all the active working group drafts. This makes it easier to + associate issues with drafts for the WG participants, without the Chairs needing to + go in as admin users and add a new component each time there's a new WG draft. + + * The 'Severity' field of an issue has a special significance if the issue type is + set to 'state' or 'task'. In that case, the Severity will be shown as an + annotation to the draft state on the regular WG status page on tools.ietf.org. + This can be useful for WG chairs to indicate more exactly the state of a WG draft, + which will otherwise simply be indicated as 'Active' on the status page, until it + is sent to the IESG for processing. + + * If issues are registered against a draft ,indicated by setting the issue's + 'component' field to the appropriate (abbreviated) draft name, the status page + will show a progress bar, indicating the total number of issues for that draft, as + well as the proportion which have been closed, and the number of still open issues. + + * Everywhere you can use wiki markup (on the wiki pages, roadmap descriptions, + etc.) you may embed a macro which shows a ticket statistics graph. Full + information about the macro is available at [http://trac-hacks.org/wiki/TicketStatsMacro]. + Briefly, the macro syntax is: + {{{ + [[TicketStats( height=250,daterange=12m,res_days=30)]] + }}} + which gives this result: [[TicketStats( height=250,daterange=12m,res_days=30)]] + +Issue tracker changes which are reflected in the WG status pages ('Severity' +annotations and issue progress bars) may take up to 1 hour to propagate from the +server which hosts the Trac instance (trac.tools.ietf.org) to the other tools servers. + +== Integration with tools.ietf.org == + +For all working groups which have an instance of Trac installed, the URL to Trac for +that WG has the form '''''!http://tools.ietf.org/wg//trac'''''. +There's also a link to the Trac issue tracker and a link to the Trac +wiki in the horizontal menu on the WG status page +'''''!http://tools.ietf.org/wg/''''' +once Trac has been installed. + +== SVN Repository == + +For each WG with a Trac instance there is also a SVN repository, with an URL of the +form '''''!https://svn.tools.ietf.org/svn/wg/'''''. Anybody can check out from +the repository, but you need to use the tools server login and password in order to +commit to the repository. + +To check out a repository with a command-line svn client, see this example for the ''hybi'' WG: + +{{{ +work/ $ svn co --username=henrik@levkowetz.com https://svn.tools.ietf.org/svn/wg/hybi/ +work/ $ cd hybi/ +hybi/ $ +}}} + +SVN also lets you check out parts of the repository tree, but for more info on +that, please see the documentation on http://subversion.apache.org/. + +To add a document to the repository, place the document in your SVN working folder, +tell SVN it should be added, and when ready, commit it to the repository: +{{{ +hybi/ $ svn add draft-foo-bar-baz.txt +hybi/ $ #... +hybi/ $ svn commit draft-foo-bar-baz.txt -m "Commit message ..." +hybi/ $ +}}} + +The IETF Trac instances use a variation of the Trac SVN hook script which is provided +with Trac. This script updates Track Issue Tickets based on keywords in the SVN +commit messages; the keywords and their use is described in SvnTracHooks. diff --git a/ietf/templates/utils/wiki/InterMapTxt b/ietf/templates/utils/wiki/InterMapTxt new file mode 100644 index 000000000..cdd997a71 --- /dev/null +++ b/ietf/templates/utils/wiki/InterMapTxt @@ -0,0 +1,72 @@ += InterMapTxt = + +== This is the place for defining InterWiki prefixes == + +This page was modelled after the MeatBall:InterMapTxt page. +In addition, an optional comment is allowed after the mapping. + + +This page is interpreted in a special way by Trac, in order to support +!InterWiki links in a flexible and dynamic way. + +The code block after the first line separator in this page +will be interpreted as a list of !InterWiki specifications: +{{{ +prefix URL [ # comment] +}}} + +By using `$1`, `$2`, etc. within the URL, it is possible to create +InterWiki links which support multiple arguments, e.g. Trac:ticket:40. +The URL itself can be optionally followed by a comment, +which will subsequently be used for decorating the links +using that prefix. + +New !InterWiki links can be created by adding to that list, in real time. +Note however that ''deletions'' are also taken into account immediately, +so it may be better to use comments for disabling prefixes. + +Also note that !InterWiki prefixes are case insensitive. + + +== List of Active Prefixes == + +[[InterWiki]] + + +---- + +== Prefix Definitions == + +{{{ +PEP http://www.python.org/peps/pep-$1.html # Python Enhancement Proposal +Trac-ML http://thread.gmane.org/gmane.comp.version-control.subversion.trac.general/ # Message $1 in Trac Mailing List +trac-dev http://thread.gmane.org/gmane.comp.version-control.subversion.trac.devel/ # Message $1 in Trac Development Mailing List + +Mercurial http://www.selenic.com/mercurial/wiki/index.cgi/ # the wiki for the Mercurial distributed SCM + +RFC http://tools.ietf.org/html/rfc$1.html # IETF's RFC $1 +DataTracker https://datatracker.ietf.org/doc/ +dt https://datatracker.ietf.org/doc/ + +# +# A arbitrary pick of InterWiki prefixes... +# +Acronym http://www.acronymfinder.com/af-query.asp?String=exact&Acronym= +C2find http://c2.com/cgi/wiki?FindPage&value= +Cache http://www.google.com/search?q=cache: +CPAN http://search.cpan.org/perldoc? +DebianBug http://bugs.debian.org/ +DebianPackage http://packages.debian.org/ +Dictionary http://www.dict.org/bin/Dict?Database=*&Form=Dict1&Strategy=*&Query= +Google http://www.google.com/search?q= +GoogleGroups http://groups.google.com/group/$1/msg/$2 # Message $2 in $1 Google Group +JargonFile http://downlode.org/perl/jargon-redirect.cgi?term= +MeatBall http://www.usemod.com/cgi-bin/mb.pl? +MetaWiki http://sunir.org/apps/meta.pl? +MetaWikiPedia http://meta.wikipedia.org/wiki/ +MoinMoin http://moinmoin.wikiwikiweb.de/ +WhoIs http://www.whois.sc/ +Why http://clublet.com/c/c/why? +c2Wiki http://c2.com/cgi/wiki? +WikiPedia http://en.wikipedia.org/wiki/ +}}} diff --git a/ietf/templates/utils/wiki/SvnTracHooks b/ietf/templates/utils/wiki/SvnTracHooks new file mode 100644 index 000000000..79c7cbd72 --- /dev/null +++ b/ietf/templates/utils/wiki/SvnTracHooks @@ -0,0 +1,77 @@ += SVN Trac Hooks = + +If the Trac Hooks for SVN has been installed for the svn repository +coupled to this Trac instance, the Key Phrases documented below may +be used in SVN commit messages to cause automatic updates and annotations +of Trac issues. + +== The trac-post-commit-hook == + +This script looks at the commit message associated with an SVN commit, +and based on the presence of a number of key phrases will add annotations +to tickets and also possibly change ticket status, for instance closing +it. + +=== Key Phrases === + +The key phrases available are: +{{{ + Fix + Fixes + Fix for + Close + Closes + + Addresses + References + Relates to + Related to + See +}}} + +=== Ticket specification === + +The specification of the ticket to act on may specify one or more +tickets, using any of the following forms: +{{{ + + , {, } + , and +}}} +and variations thereof. + +=== Ticket identification === + +The individual ticket specification +can take any of the following forms: +{{{ + # + ticket + ticket: + issue + issue: + bug + bug: +}}} + +=== Examples === + +{{{ + Clarify header normalization vs matching request headers (see #147) + + Resolve #27: fix definition of idempotent + + Note change for issue 157 (related to #157) + + Define http and https URI schemes: addresses #58, #128, #159 + + Define http and https URI schemes: addresses #58, #128, #159; + fixes #157: removed reference to RFC1900 use of IP addresses in URI. + + Resolve #140: rephrase note so that it becomes clear that the described ... +}}} + +=== Script === + +The default script installed as trac-post-commit-hook is: +http://tools.ietf.org/tools/wg-pages/svn-hook-files/trac-post-commit-hook diff --git a/ietf/templates/utils/wiki/ThisTracInstallation b/ietf/templates/utils/wiki/ThisTracInstallation new file mode 100644 index 000000000..454c3f7c8 --- /dev/null +++ b/ietf/templates/utils/wiki/ThisTracInstallation @@ -0,0 +1,94 @@ +{{{ +#!rst + +Trac Installation on tools.ietf.org +=================================== + +Background +---------- + +The Track installation used on the tools.ietf.org site is different from the +installation examples provided with Trac and on http://trac.edgewall.com. The +reason is mainly that the multi-project examples all assume that Trac +constitutes the whole of the deployed environment, rather than being part of a +greater set. This means that the examples assume that accessing the +individual projects through URLs of the form "/$some_path/trac/$projname" +makes sense, while in our case, we would like the URLs to look like +"/$some_path/$projname/trac". In the multi-project configuration, this would +make Trac always believe that the project name is 'trac' - the last path +component. + +Explored Alternatives +--------------------- + +Make Apache set ``TRAC_ENV`` dynamically +........................................ + +Tell Apache to dynamically set Trac's environment variable ``TRAC_ENV`` to the +particular value for the accessed project: + +``/etc/apache2/sites-available/tools.ietf.org``: + +:: + + ScriptAliasMatch "^/wg/[^/]+/trac(/.*)?" /usr/share/trac/cgi-bin/trac.cgi$1 + + SetEnv TRAC_ENV "/www/tools.ietf.org/tools/trac/wg/$1" + + +This doesn't work because Apache doesn't support $n replacements based on +earlier LocationMatch matches. + +Use .htaccess with default ScriptAlias +...................................... + +Maybe we could use individual .htaccess files in each WG directory to set the +``TRAC_ENV`` variable to the required value? + +``/etc/apache2/sites-available/tools.ietf.org``: + +:: + + ScriptAliasMatch "^/wg/[^/]+/trac(/.*)?" /usr/share/trac/cgi-bin/trac.cgi$1 + + +``/www/tools.ietf.org/wg/examplewg/.htaccess``: + +:: + + SetEnv TRAC_ENV "/www/tools.ietf.org/wg/examplewg/trac" + +This doesn't work because this .htaccess isn't read when ScriptAlias points to +another directory. + + +Use .htaccess with a local CGI script +..................................... + +Suppose we let ScriptAlias point to a script which is placed so that the +.htaccess file actually gets read? + +``/etc/apache2/sites-available/tools.ietf.org``: + +:: + + ScriptAliasMatch "^/wg/([^/]+)/trac(/.*)?" /www/tools.ietf.org/wg/$1/trac/index.cgi$2 + + +``/www/tools.ietf.org/wg/examplewg/.htaccess``: + +:: + + SetEnv TRAC_ENV "/www/tools.ietf.org/wg/examplewg/trac" + +This *does* work, but it is not easily adapted to a Fast-CGI solution. It is +the set-up which is currently in use, but an alternative which will permit +fast-cgi usage would be preferred - the current solution is anything but +snappy... + + + + + + +}}} diff --git a/ietf/templates/utils/wiki/TrainingMaterials b/ietf/templates/utils/wiki/TrainingMaterials new file mode 100644 index 000000000..5c1622933 --- /dev/null +++ b/ietf/templates/utils/wiki/TrainingMaterials @@ -0,0 +1,10 @@ += Training Materials = + +WG Traingin Materials go here. + +If you want to embed video, you can use the ![[Movie(,width=,height=)]] +macro to embed moves from [http://youtube.com/ YouTube]. Suggested width and height parameters: width=640,height=385. + +Example which doesn't point to an actual video: + +[[Movie(http://www.youtube.com/watch?v=g_exampleid,width=640px,height=385px)]] diff --git a/ietf/templates/utils/wiki/WikiStart b/ietf/templates/utils/wiki/WikiStart new file mode 100644 index 000000000..44bdc9ff4 --- /dev/null +++ b/ietf/templates/utils/wiki/WikiStart @@ -0,0 +1,29 @@ += Welcome to this IETF WG Trac installation = + +Trac is a '''minimalistic''' approach to '''web-based''' project management, +suitable for software and documentation projects and similar. Its goal is to +simplify effective tracking and handling of project issues, enhancements and +overall progress. + +As all Wiki pages, this page is editable, this means that you can modify the +contents of this page simply by using your web-browser. Simply click on the +"Edit this page" link at the bottom of the page. WikiFormatting will give you +a detailed description of available Wiki formatting commands. + +There is nothing in this page which isn't also covered in one of the other +wiki pages, so the first adjustment you make of this Trac installation could +be to edit the content of this page, replacing this initial text with content +appropriate to your Working Group. + +There are some aspects of this Trac installation which are specific to the +IETF environment. Those are described in IetfSpecificFeatures. + + +== Starting Points == + + * TracGuide -- Built-in Documentation + * [http://trac.edgewall.org/ The Trac project] -- Trac Open Source Project + * [http://trac.edgewall.org/wiki/TracFaq Trac FAQ] -- Frequently Asked Questions + * TracSupport -- Trac Support + +For a complete list of local wiki pages, see TitleIndex. diff --git a/ietf/utils/management/commands/create_group_wikis.py b/ietf/utils/management/commands/create_group_wikis.py new file mode 100644 index 000000000..d00ac9f58 --- /dev/null +++ b/ietf/utils/management/commands/create_group_wikis.py @@ -0,0 +1,280 @@ +# Copyright 2016 IETF Trust + +import os +import copy +import syslog +import pkg_resources +from optparse import make_option +#from optparse import make_option + +from trac.core import TracError +from trac.env import Environment +from trac.perm import PermissionSystem +from trac.ticket.model import Component, Milestone, Severity +from trac.util.text import unicode_unquote +from trac.wiki.model import WikiPage + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.template.loader import render_to_string + +import debug # pyflakes:ignore + +from ietf.group.models import Group, GroupURL +from ietf.utils.pipe import pipe + +logtag = __name__.split('.')[-1] +logname = "user.log" +syslog.openlog(logname, syslog.LOG_PID, syslog.LOG_USER) + +class Command(BaseCommand): + help = "Create group wikis for WGs, RGs and Areas which don't have one." + + option_list = BaseCommand.option_list + ( + make_option('--wiki-dir-pattern', dest='wiki_dir_pattern', help='File containing email (default: stdin)'), + ) + + def note(self, msg): + if self.verbosity > 1: + self.stdout.write(msg) + + def log(self, msg): + syslog.syslog(msg) + self.stderr.write(msg) + + # --- svn --- + + def do_cmd(self, cmd, *args): + quoted_args = [ '"%s"'%a if ' ' in a else a for a in args ] + self.note("Running %s %s ..." % (os.path.basename(cmd), " ".join(quoted_args))) + command = [ cmd, ] + list(args) + code, out, err = pipe(command) + msg = None + if code != 0: + msg = "Error %s: %s when executing '%s'" % (code, err, " ".join(command)) + self.log(msg) + return msg, out + + def svn_admin_cmd(self, *args): + return self.do_cmd(settings.SVN_ADMIN_COMMAND, *args) + + def create_svn(self, svn): + self.note(" Creating svn repository: %s" % svn) + if not os.path.exists(os.path.dirname(svn)): + msg = "Intended to create '%s', but parent directory is missing" % svn + self.log(msg) + return msg + err, out= self.svn_admin_cmd("create", svn ) + return err + + # --- trac --- + + def remove_demo_components(self, group, env): + for component in Component.select(env): + if component.name.startswith('component'): + component.delete() + + def remove_demo_milestones(self, group, env): + for milestone in Milestone.select(env): + if milestone.name.startswith('milestone'): + milestone.delete() + + def symlink_to_master_assets(self, group, env): + master_dir = settings.TRAC_MASTER_DIR + master_htdocs = os.path.join(master_dir, "htdocs") + group_htdocs = os.path.join(group.trac_dir, "htdocs") + self.note(" Symlinking %s to %s" % (master_htdocs, group_htdocs)) + os.removedirs(group_htdocs) + os.symlink(master_htdocs, group_htdocs) + + def add_wg_draft_states(self, group, env): + for state in settings.TRAC_ISSUE_SEVERITY_ADD: + self.note(" Adding severity %s" % state) + severity = Severity(env) + severity.name = state + severity.insert() + + def add_wiki_page(self, env, name, text): + page = WikiPage(env, name) + if page.time: + self.note(" ** Page %s already exists, not adding it." % name) + return + page.text = text + page.save(author="(System)", comment="Initial page import") + + def add_default_wiki_pages(self, group, env): + dir = pkg_resources.resource_filename('trac.wiki', 'default-pages') + #WikiAdmin(env).load_pages(dir) + with env.db_transaction: + for name in os.listdir(dir): + filename = os.path.join(dir, name) + name = unicode_unquote(name.encode('utf-8')) + if os.path.isfile(filename): + self.note(" Adding page %s" % name) + with open(filename) as file: + text = file.read().decode('utf-8') + self.add_wiki_page(env, name, text) + + def add_custom_wiki_pages(self, group, env): + for templ in settings.TRAC_WIKI_PAGES_TEMPLATES: + _, name = os.path.split(templ) + text = render_to_string(templ, {"group": group}) + self.note(" Adding page %s" % name) + self.add_wiki_page(env, name, text) + + def sync_default_repository(self, group, env): + repository = env.get_repository('') + if repository: + self.note(" Indexing default repository") + repository.sync() + + def create_trac(self, group): + if not os.path.exists(os.path.dirname(group.trac_dir)): + msg = "Intended to create '%s', but parent directory is missing" % group.trac_dir + self.log(msg) + return None + options = copy.deepcopy(settings.TRAC_ENV_OPTIONS) + # Interpolate group field names to values in the option settings: + for i in range(len(options)): + sect, key, val = options[i] + val = val.format(**group.__dict__) + options[i] = sect, key, val + # Try to creat ethe environment, remove unwanted defaults, and add + # custom pages and settings. + try: + env = Environment(group.trac_dir, create=True, options=options) + self.remove_demo_components(group, env) + self.remove_demo_milestones(group, env) + self.maybe_add_group_url(group, 'Wiki', settings.TRAC_WIKI_URL_PATTERN % group.acronym) + self.maybe_add_group_url(group, 'Issue tracker', settings.TRAC_ISSUE_URL_PATTERN % group.acronym) + # Use custom assets (if any) from the master setup + self.symlink_to_master_assets(group, env) + if group.type_id == 'wg': + self.add_wg_draft_states(group, env) + self.add_custom_wiki_pages(group, env) + self.add_default_wiki_pages(group, env) + self.sync_default_repository(group, env) + # Components (i.e., drafts) will be handled during components + # update later + # Permissions will be handled during permission update later. + return env + except TracError as e: + self.log("While creating trac instance for %s: %s" % (group, e)) + raise + return None + + def update_trac_permissions(self, group, env): + mgr = PermissionSystem(env) + permission_list = mgr.get_all_permissions() + permission_list = [ (u,a) for (u,a) in permission_list if not u in ['anonymous', 'authenticated']] + permissions = {} + for user, action in permission_list: + if not user in permissions: + permissions[user] = [] + permissions[user].append(action) + roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) + users = [] + for role in roles: + user = role.email.address.lower() + users.append(user) + if not user in permissions: + try: + mgr.grant_permission(user, 'TRAC_ADMIN') + self.note(" Granting admin permission for %s" % user) + except TracError as e: + self.log("While adding admin permission for %s: %s" (user, e)) + for user in permissions: + if not user in users: + if 'TRAC_ADMIN' in permissions[user]: + try: + self.note(" Revoking admin permission for %s" % user) + mgr.revoke_permission(user, 'TRAC_ADMIN') + except TracError as e: + self.log("While revoking admin permission for %s: %s" (user, e)) + + def update_trac_components(self, group, env): + components = Component.select(env) + comp_names = [ c.name for c in components ] + group_docs = group.document_set.filter(states__slug='active', type_id='draft').distinct() + group_comp = [] + for doc in group_docs: + if not doc.name.startswith('draft-'): + self.log("While adding components: unexpectd %s group doc name: %s" % (group.acronym, doc.name)) + continue + name = doc.name[len('draft-'):] + if name.startswith('ietf-'): + name = name[len('ietf-'):] + elif name.startswith('irtf-'): + name = name[len('ietf-'):] + if name.startswith(group.acronym+'-'): + name = name[len(group.acronym+'-'):] + group_comp.append(name) + if not name in comp_names and not doc.name in comp_names: + self.note(" Group draft: %s" % doc.name) + self.note(" Adding component %s" % name) + comp = Component(env) + comp.name = name + comp.owner = "%s@ietf.org" % doc.name + comp.insert() + + def maybe_add_group_url(self, group, name, url): + urls = [ u for u in group.groupurl_set.all() if name.lower() in u.name.lower() ] + if not urls: + self.note(" adding %s %s URL ..." % (group.acronym, name.lower())) + group.groupurl_set.add(GroupURL(group=group, name=name, url=url)) + + def add_custom_pages(self, group, env): + for template_name in settings.TRAC_WIKI_PAGES_TEMPLATES: + pass + + def add_custom_group_states(self, group, env): + for state_name in settings.TRAC_ISSUE_SEVERITY_ADD: + pass + + # -------------------------------------------------------------------- + + def handle(self, *filenames, **options): + self.verbosity = options['verbosity'] + self.errors = 0 + self.wiki_dir_pattern = options.get('wiki_dir_pattern', settings.TRAC_WIKI_DIR_PATTERN) + + if isinstance(self.verbosity, (type(""), type(u""))) and self.verbosity.isdigit(): + self.verbosity = int(self.verbosity) + + if not os.path.exists(os.path.dirname(self.wiki_dir_pattern)): + raise CommandError('The Wiki base direcory specified for the wiki directories (%s) does not exist.' % os.path.dirname(self.wiki_dir_pattern)) + + groups = Group.objects.filter( + type__slug__in=['wg','rg','area'], + state__slug='active' + ).order_by('acronym') + + for group in groups: + try: + self.note("Processing group %s" % group.acronym) + group.trac_dir = self.wiki_dir_pattern % group.acronym + group.svn_dir = settings.TRAC_SVN_DIR_PATTERN % group.acronym + + if not os.path.exists(group.svn_dir): + err = self.create_svn(group.svn_dir) + self.errors += 1 if err else 0 + + if not os.path.exists(group.trac_dir): + trac_env = self.create_trac(group) + self.errors += 1 if not trac_env else 0 + else: + trac_env = Environment(group.trac_dir) + + if not trac_env: + continue + + self.update_trac_permissions(group, trac_env) + self.update_trac_components(group, trac_env) + + except Exception as e: + self.errors += 1 + self.log("While processing %s: %s" % (group.acronym, e)) + raise + + if self.errors: + raise CommandError("There were %s failures in WG Trac creation, see syslog %s for details." % (self.errors, logname)) diff --git a/ietf/utils/pipe.py b/ietf/utils/pipe.py index d4f7c6087..845bc901e 100644 --- a/ietf/utils/pipe.py +++ b/ietf/utils/pipe.py @@ -24,7 +24,7 @@ def pipe(cmd, str=None): err = pipe.childerr.read() break if len(out) >= MAX: - err = "Output exceeds %s bytes and has been truncated" + err = "Output exceeds %s bytes and has been truncated" % MAX break return (code, out, err) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 6bf4d7336..b3845c3f1 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- import os.path import types -#import json -#from pathlib import Path +import shutil +from StringIO import StringIO +from pipe import pipe from textwrap import dedent from email.mime.text import MIMEText @@ -10,6 +11,7 @@ from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from django.conf import settings +from django.core.management import call_command from django.template import Context from django.template.defaulttags import URLNode from django.templatetags.static import StaticNode @@ -21,7 +23,9 @@ import debug # pyflakes:ignore import ietf.urls from ietf.utils.management.commands import pyflakes from ietf.utils.mail import send_mail_text, send_mail_mime, outbox +from ietf.utils.test_data import make_test_data from ietf.utils.test_runner import get_template_paths +from ietf.group.models import Group class PyFlakesTestCase(TestCase): @@ -186,6 +190,46 @@ class TemplateChecksTestCase(TestCase): settings.DEBUG = saved_debug +class TestWikiGlueManagementCommand(TestCase): + + def setUp(self): + self.wiki_dir_pattern = os.path.abspath('tmp-wiki-dir-root/%s') + if not os.path.exists(self.wiki_dir_pattern): + os.mkdir(os.path.dirname(self.wiki_dir_pattern)) + + def tearDown(self): + shutil.rmtree(os.path.dirname(self.wiki_dir_pattern)) + + def test_wiki_create_output(self): + make_test_data() + groups = Group.objects.filter( + type__slug__in=['wg','rg','area'], + state__slug='active' + ).order_by('acronym') + out = StringIO() + call_command('create_group_wikis', stdout=out, verbosity=2, wiki_dir_pattern=self.wiki_dir_pattern) + command_output = out.getvalue() + for group in groups: + self.assertIn("Processing group %s" % group.acronym, command_output) + # Do a bit of verification using trac-admin, too + admin_code, admin_output, admin_error = pipe('trac-admin %s permission list' % (self.wiki_dir_pattern % group.acronym)) + self.assertEqual(admin_code, 0) + roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) + for role in roles: + user = role.email.address.lower() + self.assertIn("Granting admin permission for %s" % user, command_output) + self.assertIn(user, admin_output) + docs = group.document_set.filter(states__slug='active', type_id='draft') + for doc in docs: + name = doc.name + name = name.replace('draft-','') + name = name.replace(doc.stream_id+'-', '') + name = name.replace(group.acronym+'-', '') + self.assertIn("Adding component %s"%name, command_output) + for page in settings.TRAC_WIKI_PAGES_TEMPLATES: + self.assertIn("Adding page %s" % os.path.basename(page), command_output) + self.assertIn("Indexing default repository", command_output) + ## One might think that the code below would work, but it doesn't ... # def list_static_files(path): diff --git a/ietf/wsgi.py b/ietf/wsgi.py index 28c5b2b21..3a4cb3edb 100644 --- a/ietf/wsgi.py +++ b/ietf/wsgi.py @@ -44,7 +44,7 @@ import sys path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Virtualenv support -virtualenv_activation = os.path.join(path, "bin", "activate_this.py") +virtualenv_activation = os.path.join(path, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) diff --git a/requirements.txt b/requirements.txt index 26e2ecad7..59008acc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,6 +34,7 @@ pyzmail>=1.0.3 selenium>=2.42 six>=1.8.0 tqdm>=3.7.0 +Trac>=1.0.10 Unidecode>=0.4.18 #wsgiref>=0.1.2 xml2rfc>=2.5.