diff --git a/6.130.2.dev0/.bowerrc b/6.130.2.dev0/.bowerrc new file mode 100644 index 000000000..f2571d7ff --- /dev/null +++ b/6.130.2.dev0/.bowerrc @@ -0,0 +1,3 @@ +{ + "registry": "https://registry.bower.io" +} diff --git a/6.130.2.dev0/.editorconfig b/6.130.2.dev0/.editorconfig new file mode 100644 index 000000000..63bf7a66d --- /dev/null +++ b/6.130.2.dev0/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig helps developers define and maintain consistent +# coding styles between different editors and IDEs +# editorconfig.org + +root = true + +# Settings for IETF datatracker + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +# to avoid tripping Henrik's commit hook: +trim_trailing_whitespace = false +insert_final_newline = false diff --git a/6.130.2.dev0/.eslintrc.js b/6.130.2.dev0/.eslintrc.js new file mode 100644 index 000000000..349bc61b8 --- /dev/null +++ b/6.130.2.dev0/.eslintrc.js @@ -0,0 +1,19 @@ +module.exports = { + rules: { + indent: [2, 4], + camelcase: 0, + "require-jsdoc": 0, + quotes: [2, "double"], + "no-multiple-empty-lines": [2, {max: 2}], + "quote-props": [2, "as-needed"], + "brace-style": [2, "1tbs", {allowSingleLine: true}] + }, + env: { + browser: true, + jquery: true + }, + globals: { + d3: true + }, + extends: "google" +}; diff --git a/6.130.2.dev0/.gitignore b/6.130.2.dev0/.gitignore new file mode 100644 index 000000000..e820a8f49 --- /dev/null +++ b/6.130.2.dev0/.gitignore @@ -0,0 +1,37 @@ +/.coverage +/.mypy_cache +/.project +/.pydevproject +/.Python +/.settings +/.tmp +/.factoryboy_random_state +/attic +/bin +/etc +/env +/ghostdriver.log +/htmlcov +/include +/INSTALL.pdf +/latest-coverage.json +/lib +/local +/media +/mergelog +/mergequeue +/minimum-requirements.txt +/pip-selfcheck.json +/selenium +/settings_local.py +/share +/static +/testresult +/tmp +/tmp-nomcom-public-keys-dir +/trunk27 +/trunk36 +/trunk37 +/unix.tag +*.pyc +__pycache__ diff --git a/6.130.2.dev0/.pylintrc b/6.130.2.dev0/.pylintrc new file mode 100644 index 000000000..c9a33fcec --- /dev/null +++ b/6.130.2.dev0/.pylintrc @@ -0,0 +1,408 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS,migrations + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint_django,pylint_common + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. This option is deprecated +# and it will be removed in Pylint 2.0. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +#disable=C,import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,unused-wildcard-import,locally-disabled,mixed-indentation +disable=W,C,R,I,no-member + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=parseable + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". This option is deprecated +# and it will be removed in Pylint 2.0. +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/6.130.2.dev0/INSTALL b/6.130.2.dev0/INSTALL new file mode 100644 index 000000000..3aad0c1c3 --- /dev/null +++ b/6.130.2.dev0/INSTALL @@ -0,0 +1,283 @@ +============================================================================== + IETF Datatracker +============================================================================== + +------------------------------------------------------------------------------ + Installation Instructions +------------------------------------------------------------------------------ + + + +General Instructions for Deployment of a New Release +==================================================== + + 1. In order to fetch a new release of the django datatracker code, simply + check out the appropriate tag from svn:: + + svn co http://svn.tools.ietf.org/svn/tools/ietfdb/tags/$releasenumber + + 2. Don't forget to copy $releasenumber/ietf/settings_local.py from the + old release to the new one; otherwise things won't work! + :: + + cp $oldreleasenumber/ietf/settings_local.py $releasenumber/ietf/ + + 3. Change into the directory of the new release:: + + cd $releasenumber + + It is recommended set up a Python virtual environment + under $releasenumber/env/:: + + virtualenv env # optional + source env/bin/activate # optional + + 4. Install requirements (make sure your pip is reasonably fresh first). + The following will install required python modules locally if you + are using a virtualenv, or globally if you are not:: + + pip install -r requirements.txt + + 5. Move static files to the appropriate direcrory for serving via CDN:: + + ietf/manage.py collectstatic + + 6. Run migrations. Some migrations create files which need to be writeable + by the web servers, so make sure that you're running as wwwrun when + doing this:: + + sudo su wwwrun -s /bin/bash + source env/bin/activate + ietf/manage.py migrate + ^D + + 7. Run some basic datatracker system checks:: + + ietf/manage.py check + + 8. Back out one directory level, then re-point the 'web' symlink:: + + cd .. + rm ./web + ln -s $releasenumber web + + 9. Reload apache:: + + sudo service apache2 reload + + 10. It's now also a good idea to go to the datatracker front page:: + + http://datatracker.ietf.org/ + + and then check that it's alive and kicking, and displaying the new release + number at the bottom of the left-side menubar :-) + + 11. If things **aren't** cool, revert the symlink step, re-pointing the + symlink to the release that was running before the new release, and restart + apache again to roll back to that. + + +Installing from Scratch +======================= + +In addition to the new release deployment instructions above, the +settings_local.py file has to be set up properly, and Apache has to be +configured. Since the IETF datatracker is only intended to be deployed from +scratch once, these instructions don't cover this scenario in any more detail. +The general Django depoloyment instructions are relevant, however. + + +Patching a Production Release +============================= + +Sometimes it can prove necessary to patch an existing release. The following +process should be used: + + 1. Code and test the patch on a development system copy of the production + release which has no other code changes (or on trunk, with no uncommitted + code changes, if it's sufficiently close). + + 2. Produce a patch file, named with date and subject:: + + ~/src/ietfdb/working $ svn diff > 2013-03-25-ballot-calculation.patch + + 3. Move the patch file to the production server, and place it in + '/a/www/ietf-datatracker/patches/' + + 4. Make a recursive copy of the production code to a new directory, named + with a patch number. Assuming the production code is in 4.43/, and we + have web -> 4.43/:: + + /a/www/ietf-datatracker $ rsync -a web/ 4.43.p1/ + + (you could use 'cp -rp' instead, but rsync seems to do this faster). + + 5. Apply the patch:: + + /a/www/ietf-datatracker $ cd 4.43.p1/ + /a/www/ietf-datatracker/4.43.p1 $ patch -p0 \ + < ../patches/2013-03-25-ballot-calculation.patch + + This should not produce any messages about failing to apply any chunks; + if it does, you probably should go back to 1. and figure + out why. + + 6. Edit ``.../ietf/__init__.py`` in the new patched release to indicate the patch + version in the ``__patch__`` string and current date and time in the ``__date__`` + string. + + 7. Change the 'web' symlink, reload etc. as described in + `General Instructions for Deployment of a New Release`_ . + + +Additional Version-Specific Instructions +======================================== + +Version 4.50 +------------ + +Before you run step 3 (migration) of the general instructions, please run some specific +initial migrations with the a --fake argument: + + cd $releasenumber + PYTHONPATH=$PWD ietf/manage.py migrate group 0001 --fake + cd .. + + +Version 4.42 +------------ + + - In order to serve the secretariat tools static pages (such as image, css and js + files) the exceptions to which urls need to be handled by the python-handler + must be updated. + + In the datatracker test server, the following configuration has been used to + make apache handle the static files, and Django the dynamic pages. The new + part is the which mentions /secr/. Adapt as needed for ietfa:: + + ---------------- + # Logging and document root settings omitted + + + PythonPath "['/srv/www/ietfdb'] + sys.path" + SetHandler python-program + PythonHandler django.core.handlers.modpython + SetEnv DJANGO_SETTINGS_MODULE ietf.settings + PythonDebug On + + + + SetHandler None + + + # New for secretariat tools: + + SetHandler None + + + + AuthType Digest + AuthName "IETF" + AuthUserFile /var/local/loginmgr/digest + AuthGroupFile /var/local/loginmgr/groups + AuthDigestDomain http://tools.ietf.org/ + Require valid-user + + + # Caching and compression settings omitted + ---------------- + +Version 4.40 +------------ + + - (DONE) Add ianasync user with an auth role in the "iana" group and an + rfceditorsync user with an auth role in the "rfceditor" group (don't + think Group(acronym="rfceditor") exists yet); IANA and RFC Editor need + to know the passwords for the poke mechanism + + + - (DONE) Make sure mailing list for iab-stream@iab.org is up (since we're now + emailing that) + + - (DONE) Set rfc_must_published_later_than date in bin/iana-protocols-updates to today + + - (DONE) Run the 3 new doc South migrations + + - (DONE) New polling scripts, to be run every hour:: + + web/ietf/bin/iana-changes-updates + web/ietf/bin/iana-protocols-updates + web/ietf/bin/rfc-editor-index-updates (replaces mirror_rfc_index) + web/ietf/bin/rfc-editor-queue-updates (replaces mirror_rfc_queue) + + - (DONE) Import old events from IANA:: + + bin/iana-changes-updates --from="2005-01-01 00:00:00" --to="2013-01-31 00:00:00" --no-email + + - (DONE) Pipe IANA review emails to the datatracker. There used to be an action to pipe + such mails to henrik@levkowetz.com, for testing this feature, but I haven't seen + any in a little while, so I don't know if this has broken. Anyway, the iana review + emails should be piped into:: + + /www/ietf-datatracker/web/ietf/bin/iana-review-email + + + - (DONE) Tell IANA we're doing this for real now. + + +Version 4.34 +------------ + +The migration step you do as a part of the release sequence is going to take +quite some time -- probably minutes. It will generate some output while it's +working, too. As long as it doesn't halt and say that something failed or +gave an error, this is as expected, and when it terminates, you should be OK +to continue. + +Version 4.21 +------------ + +This release will you to run migrations before moving the link to the new +version and doing the apache reload. I know you have a routine for the steps +needed to deploy a new release by now, but thought I'd mention it, anyway. + +If there is any problem at all doing the migrations, then you'll need to +do a fake initial migration, as follows:: + + web $ PYTHONPATH=PWD ietf/manage.py migrate --fake meeting 0001 + +and then to the regular migration again. + +Version 4.20 +------------ + +Some one-time actions that need to be taken are as follows:: + + Assuming that the release has been checked out in /a/www/ietf-datatracker/4.20: + + cd /a/www/ietf-datatracker/4.20 + + PYTHONPATH=$PWD ietf/manage.py migrate --fake doc 0001 + PYTHONPATH=$PWD ietf/manage.py migrate --fake name 0001 + + PYTHONPATH=$PWD ietf/manage.py dbshell <<< "delete from django_content_type where app_label='doc' + and model='groupballotpositiondocevent';" + + PYTHONPATH=$PWD ietf/manage.py migrate doc || { \ + PYTHONPATH=$PWD ietf/manage.py dbshell <<< 'CREATE TABLE `doc_groupballotpositiondocevent` + (`block_comment` longtext NOT NULL, `comment` longtext NOT NULL, + `ad_id` integer NOT NULL, `comment_time` datetime NULL, + `block_comment_time` datetime NULL, `pos_id` varchar(8) NOT NULL DEFAULT "norecord", + `docevent_ptr_id` integer NOT NULL PRIMARY KEY);' + PYTHONPATH=$PWD ietf/manage.py dbshell <<< 'DROP TABLE `doc_ballottype` CASCADE;' + PYTHONPATH=$PWD ietf/manage.py dbshell <<< 'DROP TABLE `doc_ballottype_positions` CASCADE;' + PYTHONPATH=$PWD ietf/manage.py dbshell <<< 'DROP TABLE `doc_ballotdocevent` CASCADE;' + PYTHONPATH=$PWD ietf/manage.py dbshell <<< 'ALTER TABLE `doc_ballotpositiondocevent` + DROP COLUMN `ballot_id` CASCADE;' + } + + PYTHONPATH=$PWD ietf/manage.py migrate doc + PYTHONPATH=$PWD ietf/manage.py migrate name + PYTHONPATH=$PWD python ietf/wgcharter/migrate.py | tee -a ~/charter-migration.log + + diff --git a/6.130.2.dev0/LICENSE b/6.130.2.dev0/LICENSE new file mode 100644 index 000000000..6ffb15e4b --- /dev/null +++ b/6.130.2.dev0/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2008,2018, The IETF Trust +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/6.130.2.dev0/PLAN b/6.130.2.dev0/PLAN new file mode 100644 index 000000000..a48fa30fc --- /dev/null +++ b/6.130.2.dev0/PLAN @@ -0,0 +1,200 @@ +================ +Datatracker PLAN +================ + +Updated: $Date$ + +Planned work in rough order +=========================== + +* Transition to Django 2.2 (via 2.0 and 2.1). This depends on Python 3.x. + Security updates to Django 1.11 will cease around April 2020. + +* Transition to PostgreSQL. This will make it easier to start using + timezone-aware timestamps throughout the code, which will make it easy + to present localized times on web-pages. It will also provide additional + tools for performance analysis + +* Investigate making RFCs first-class document objects to faciliate being + able to model BCPs that represent groups of RFCs properly. Then fix the rfc sync + script to update BCP aliases when changes occur changes (a BCP points to new + RFC numbers) + +* Polish the htmlization pages, making the style identical with tools.ietf.org. + +* Investigate adding code to trigger test exceptions for template variables + that don't successfully resolve. A small trial run of this has indicated + that most instances that trigger such exceptions would benefit from better + resulution -- i.e., in the trial, it didn't seem that we to any greater + extent have template code that is written to rely on failure to resolve + rendering as nothing. + +* Add django pagination to selected pages with long lists of results in order + return results to the user faster. + +* Change our web asset handling from the current bower-based (which has been + deprecated and gradually more and more unworkable) to a yarn-based setup. + Short-time the effect of not having the web asset updates in mkrelease is + negigible, but long-term the lack of this means that we may miss + security-updates to web assets. + +* (EXTERNAL BID) Reworked UI and refactored backend for the scretariat meeting + scheduling tool. + +* Add support for internationalised email addresses according to RFC 6531 + when sending email (this is not supported by smtplib under Python 2.7, + so will need python 3.x). + +* Test suite improvements. Reduce and seek to eliminate causes of differences + in code coverage between runs in the same and in different environments. + +* Rework email subjects for messages relating to documents to consistently + start with the document name. + +* Revisit RFC author ordering and author information based on rfc-index.xml + +* Revisit photo uploads: Add photo upload for people with roles. Add + photos to wg pages and group overview pages. + +* Revisit floorplans: Add room coordinate input tool (javascript). + Add NOC object annotation possibilities. Add support for break areas + on multiple floors (probably means modelling break areas the same way + as other sessions, and removing the break_area char_field on Meeting. + +* Add the ability to volunteer for Nomcom when logged in to the datatracker. + +* Transition to using timezone-aware timestamps. Migrate database timestamps + to be timezone-aware. This means converting all timestamps to PST8PDT, + except for meeting-related timestamps, which use the meeting.time_zone. + Once done, set USE_TZ to True in settings. + +* Add support for document shepherding reports, possibly re-using or + generalising some of the review plumbing. Check with IESG for details. + +* Performance analysis of database table and index setup + +* Refactor Document and types into Document subclasses, with conditional code + in views and utilities moved into models and overridden models where + handling differs between document types. + +* When draft XML source is available, take references from that instead + of extracting them from the text version. + +* Integrate better author data for RFCs from either the RFC Editor or from + running the author extraction script on newly published RFCs. + +* Add an FK to person from author, and use that instead of the unknown-email-* + addresses to connect documents to authors when no email address is available. + (we still need to keep author email addresses in order to know which + address to associate with a given authorship, when we have an address). + +* Review places where we display persons' names but link to email addresses. + Some of those probably should link to the person's profile page, instead. + + +Notes +=========================== + +* Small nomcom refactoring: associate Feedback records with Person + instead of User. + +* Consistency fix: in settings.py, name directory settings consistently with + ..._DIR; reserving ..._PATH for settings with PATH semantics (':'-separated lists + of directories) + +* For documents with Yang modules, add links to the extracted modules (possibly in + multiple formats -- pyang can generate a large number of alternative formats) + +* When we get to the point where we can produce pdf from xml+media, remove the + pdf upload possibility -- it's a definite attack vector. Possibly start scanning + pdf files for /JS and /JavaScript (and workarounds to hide Javascript) + +* Add one or more API description pages with examples of common queries + +* Add role lists and additional relevant information to the personal profile + pages + +* Increase the requirements on valid email addresses in draft submissions -- + contacting authors is hard if they don't provide an email address. + +* Make it possible to let time run backwards in the database (creation timestamps, + universal changelog) + +* Change slugs which were limited by the former 8-character slug length limit + to be more readable (needs both code changes and migrations). + +* Additional charts, various statistics views. + +* For graphing and mapping various numbers relative to country population, + integrate population data from the world bank, + http://databank.worldbank.org/data/reports.aspx?source=2&series=SP.POP.TOTL&country= + (or some other source, if a better is found). WorldOmeters has running counters, + but ask for a lot of money for even one. + +* Notable New Django Features in 1.8: + + + New data types: UUIDField, DurationField + + Query Expressions (enhanced expressivity, conditionals, database functions) + + Improved TestCase fixture loading, additional .setUpTestData() method. + - Support for alternative template engines + - Security enhancements (request/response cycle middleware) + - PostgreSQL-specific fields + +* Notable New Django Features in 1.9: + + + New on_commit() hook, for post-commit actions, for instance sending email + + Password validation option with pluggable validators + + Running tests in parallel + + DateTimeFields can be queried with date + - Permission mixins for class-based views + - New admin style + + +* WG document state slug renaming. + + adopt-wg -> adopted-by-wg + c-adopt -> candidate-for-wg-adoption + chair-w -> waiting-for-chair-goahead + dead -> dead-wg-draft + info -> adopted-for-wg-info-only + parked + sub-pub -> submitted-for-publication + wg-cand -> wg-candidate + wg-doc -> wg-document + wg-lc -> wg-last-call + writeupw -> waiting-for-shepherd-writeup + +* Clean up and normalise the use of canceled / cancelled. User-visible + strings should already be all CANCELLED except when manually entered. + +* DjangoCon Europe 2017 Notes: + + - Add tracing of Django Query methods from code through templates to the + sql_queries list provided by 'django.template.context_processors.debug' + (DONE) + + - Consider rewriting user switches using feature flags, for instance with + gargoyle. + + - There is now a Django-REST-Framework add-on app which makes it easier to + do something similar with DRF as with Tastypie. As Tastypie is not being + actively maintained, and DRF seems to have better performance, consider + building /api/v2 using DRF and drf-schema-adapter / drf-auto-endpoint. + + - Consider adding JWT (RFC7519) support for /api/v1 /api/v2, to generate an + access token from a login, and use that instead of session support for + access control to access limited endpoints. + + - Once we're on Django 3.5, start using static type annotations to improve + early discovery of incorrect function/method usage. There are add-on + files avaliable for Django which provide type annotation for Django + functions and methods, and the Python stdlib is type annotated starting + with Python 3.5. Check static typing violations with 'mypy'. + + - Consider providing a user-selectable option to import photos from + gravatar. + + - Consider using django-12factor to apply part of the 12-factor app + philosophy: https://12factor.net/ + + diff --git a/6.130.2.dev0/README-CDN.rst b/6.130.2.dev0/README-CDN.rst new file mode 100644 index 000000000..f4f194289 --- /dev/null +++ b/6.130.2.dev0/README-CDN.rst @@ -0,0 +1,183 @@ +================================================================================ + Serving Static Datatracker Files via a CDN +================================================================================ + +Intro +===== + +With release 6.4.0, the way that the static files used by the datatracker are +handled changes substantially. Static files were previously versioned under a +top-level ``static/`` directory, but this is not the case any more. External +files (such as for instance ``jquery.min.js``) are now placed under +``ietf/externals/static/`` and updated using a tool called bower_, while +datatracker-specific files (images, css, js, etc.) are located under +``ietf/static/ietf/`` and ``ietf/secr/static/secr/`` respectively. + +The following sections provide more details about handling of internals, +externals, and how deployment is done. + + +Serving Static Files via CDN +============================ + +Production Mode +--------------- + +If resources served over a CDN and/or with a high max-age don't have different +URLs for different versions, then any component upgrade which is accompanied +by a change in template functionality will have a long transition time +during which the new pages are served with old components, with possible +breakage. We want to avoid this. + +The intention is that after a release has been checked out, but before it is +deployed, the standard django 'collectstatic' management command will be +run, resulting in all static files being collected from their working +directory location and placed in an appropiate location for serving via CDN. +This location will have the datatracker release version as part of its URL, +so that after the deployment of a new release, the CDN will be forced to fetch +the appropriate static files for that release. + +An important part of this is to set up the ``STATIC_ROOT`` and ``STATIC_URL`` +settings appropriately. In 6.4.0, the setting is as follows in production +mode:: + + STATIC_URL = "https://www.ietf.org/lib/dt/%s/"%__version__ + STATIC_ROOT = CDN_ROOT + "/a/www/www6s/lib/dt/%s/"%__version__ + +The result is that all static files collected via the ``collectstatic`` +command will be placed in a location served via CDN, with the release +version being part of the URL. + +Development Mode +---------------- + +In development mode, ``STATIC_URL`` is set to ``/static/``, and Django's +``staticfiles`` infrastructure makes the static files available under that +local URL root (unless you set +``settings.SERVE_CDN_FILES_LOCALLY_IN_DEV_MODE`` to ``False``). It is not +necessary to actually populate the ``static/`` directory by running +``collectstatic`` in order for static files to be served when running +``ietf/manage.py runserver`` -- the ``runserver`` command has extra support +for finding and serving static files without running collectstatic. + +In order to work backwards from a file served in development mode to the +location from which it is served, the mapping is as follows:: + + ============================== ============================== + Development URL Working copy location + ============================== ============================== + localhost:8000/static/ietf/* ietf/static/ietf/* + localhost:8000/static/secr/* ietf/secr/static/secr/* + localhost:8000/static/* ietf/externals/static/* + ============================== ============================== + +Handling of External Javascript and CSS Components +================================================== + +In order to make it easy to keep track of and upgrade external components, +these are now handled by a tool called ``bower``, via a new management +command ``bower_install``. Each external component is listed in a file +``ietf/bower.json``. In order to update the version of a component listed in +``ietf/bower.json``, or add a new one, you should edit ``bower.json``, and +then run the management command:: + + $ ietf/manage.py bower_install + +(Not surprisingly, you need to have bower_ installed in order to use this +management command.) + +That command will fetch the required version of each external component listed +in ``bower.json`` (actually, it will do this for *all* ``bower.json`` files +found in the ``static/`` directories of all ``INSTALLED_APPS`` and the +directories in ``settings.STATICFILES_DIRS``), saving them temporarily under +``.tmp/bower_components/``; it will then extract the relevant production +``js`` and ``css`` files and place them in an appropriately named directory +under ``ietf/externals/static/``. The latter location is taken from +``COMPONENT_ROOT`` in ``settings.py``. + +Managing external components via bower has the additional benefit of +managing dependencies -- components that have dependencies will pull in +these, so that they also are placed under ``ietf/externals/static/``. +You still have to manually add the necessary stylesheet and/or javascript +references to your templates, though. + +The ``bower_install`` command is not run automatically by ``bin/mkrelease``, +since it needs an updated ``bower.json`` in order to do anything interesting. +So when you're intending to update an external web asset to a newer version, +you need to edit the ``bower.json`` file, run ``manage.py bower_install``, +verify that the new version doesn't break things, and then commit the new +files under ``ietf/externals/static/`` and the updated ``bower.json``. + +.. _bower: http://bower.io/ + +The ``ietf/externals/static/`` Directory +----------------------------------------- + +The directory ``ietf/externals/static/`` holds a number of subdirectories +which hold distribution files for external client-side components, collected +by ``bower_install`` as described above. Currently +(01 Aug 2015) this means ``js`` and ``css`` components and fonts. + +These components each reside in their own subdirectory, which is named with +the component name:: + + henrik@zinfandel $ ls -l ietf/externals/static/ + total 40 + drwxr-xr-x 6 henrik henrik 4096 Jul 25 15:25 bootstrap + drwxr-xr-x 4 henrik henrik 4096 Jul 25 15:25 bootstrap-datepicker + drwxr-xr-x 4 henrik henrik 4096 Jul 25 15:25 font-awesome + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:25 jquery + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:25 jquery.cookie + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:24 ptmono + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:24 ptsans + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:24 ptserif + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:25 select2 + drwxr-xr-x 2 henrik henrik 4096 Jul 25 15:25 select2-bootstrap-css + +The ``pt*`` fonts are an exception, in that there is no bower component +available for these fonts, so they have been put in place manually. + + +Handling of Internal Static Files +================================= + +Previous to this release, internal static files were located under +``static/``, mixed together with the external components. They are now +located under ``ietf/static/ietf/`` and ``ietf/secr/static/secr``, and will be +collected for serving via CDN by the ``collectstatic`` command. Any static +files associated with a particular app will be handled the same way (which +means that all ``admin/`` static files automatically will be handled correctly, too). + +Handling of Customised Bootstrap Files +====================================== + +We are using a customised version of Bootstrap_, which is handled specially, +by a SVN externals definition in ``ietf/static/ietf``. That pulls the content +of the ``bootstrap/dist/`` directory (which is generated by running ``grunt`` +in the ``bootstrap/`` directory) into ``ietf/static/ietf/bootstrap``, from +where it is collected by ``collectstatic``. + +Changes to Template Files +========================= + +In order to make the template files refer to the correct versioned CDN URL +(as given by the STATIC_URL root) all references to static files in the +templates have been updated to use the ``static`` template tag when referring +to static files. This will automatically result in both serving static files +from the right place in development mode, and referring to the correct +versioned URL in production mode and the simpler ``/static/`` urls in +development mode. + +.. _bootstrap: http://getbootstrap.com/ + +Deployment +========== + +During deployment, it is now necessary to run the management command:: + + $ ietf/manage.py collectstatic + +before activating a new release. + +The deployment ``README`` file at ``/a/www/ietf-datatracker/README`` has been +updated accordingly. diff --git a/6.130.2.dev0/README.datatracker b/6.130.2.dev0/README.datatracker new file mode 100644 index 000000000..5c1a77b28 --- /dev/null +++ b/6.130.2.dev0/README.datatracker @@ -0,0 +1,35 @@ +The "new" datatracker uses Twitter Bootstrap for the UI. + +Get familiar with http://getbootstrap.com/getting-started/ and use those +UI elements instead of cooking up your own. + +We have some site-wide customization applied to the bootstrap version we keep +in bootstrap/ (from which the minified dist version is built); it modifies +some stuff under less/ + +We also apply some additional customizations in static/css/ietf.css; we +should eventually move that under bootstrap/less/ if possible. (ietf.css was +what Lars used initially for customization with an unmodified bootstrap.) + +Some ground rules: + +* Think hard before tweaking the bootstrap CSS, it will make it harder to + upgrade to future releases. + +* No ').appendTo('head') + + $('