From ddcd80766e48c6715c0186de729e57166f51629c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 4 Jun 2021 18:42:01 +0000 Subject: [PATCH] Merged in [19071] from mark@painless-security.com: Update the mkdevbranch script for new Trac location, don't clobber existing paths The script has been cleaned up, adjusted to avoid clobbering an SVN path that already exists when running it for a single sprinter login, and run with the new locations for the Trac system. This is a decently huge rewrite of the script. It's pulled a lot of logic into functions, excised the python code into its own file, etc. - Legacy-Id: 19076 Note: SVN reference [19071] has been migrated to Git commit 537b76eb76170d69036999f8a8ef20fd81caca93 --- bin/mkdevbranch | 279 +++++++++++++++++++++++++++----------------- bin/sprintcoders.py | 21 ++++ bin/update | 10 +- 3 files changed, 198 insertions(+), 112 deletions(-) create mode 100644 bin/sprintcoders.py diff --git a/bin/mkdevbranch b/bin/mkdevbranch index 066a798e4..7a2ff5853 100755 --- a/bin/mkdevbranch +++ b/bin/mkdevbranch @@ -3,6 +3,7 @@ version=0.24 program=${0##*/} progdir=${0%/*} +svn_url_base="https://svn.ietf.org/svn/tools/ietfdb" if [ "$progdir" = "$program" ]; then progdir="."; fi # ---------------------------------------------------------------------- @@ -24,10 +25,14 @@ DESCRIPTION EOF echo -e "OPTIONS" - if [ "$(uname)" = "Linux" ]; then - egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/' + if [ "$(uname)" = "Linux" ]; then + # shellcheck disable=SC2086 + # shellcheck disable=SC2016 + grep -E "^\s+[-][A-Za-z| -]+\*?\)\s+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | $sed -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/' else - egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /' + # shellcheck disable=SC2086 + # shellcheck disable=SC2016 + grep -E "^\s+[-][A-Za-z| -]+\*?\)\s+[A-Za-z].+#" "$0" | $sed 's/\|.*\$2[^#]*#/ /' | $sed 's/\|.*\)[^#]*#/ /' fi cat < /dev/stderr + echo -e "\n$program: error: $*" >/dev/stderr exit 1 } -function warn() { - echo "$program: Warning: $*" 1>&2; +function warn() { + echo "$program: Warning: $*" 1>&2 } - -function note() { +function note() { if [ -n "$VERBOSE" ]; then echo -e "$*"; fi } # ---------------------------------------------------------------------- function version() { - echo -e "$program $version" + echo -e "$program $version" } # ---------------------------------------------------------------------- -trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR +function check_svn_path_exists() { + local __resultvar=$1 + local __path=$2 + local myresult + + svn info "${__path}" >/dev/null 2>&1 + myresult=$? + + # shellcheck disable=SC2086 + eval $__resultvar="'$myresult'" +} + +function mksvndir() { + who=$1 + if [ "$2" ]; then dir=$2; else dir=$who; fi + check_svn_path_exists exists "${svn_url_base}/personal/$dir" + # shellcheck disable=SC2154 + if [ "$exists" != "0" ]; then + $do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir" + $do svn mkdir "${svn_url_base}/personal/$dir" -m "Personal SVN dir for $who, for IETF datatracker code" + else + echo "Repository area personal/$dir is already in place." + fi +} + +function mksvntarget() { + local who=$1 + local target=$2 + local source=$3 + local email=$4 + local name=$5 + + check_svn_path_exists exists "${svn_url_base}/personal/$who/$target" + if [ "$exists" != "0" ]; then + $do echo " creating $target branch for $who ($name)." + $do svn cp "${svn_url_base}/$source" \ + "${svn_url_base}/personal/$who/$target/" \ + -m "New branch for $target" + $do echo "New branch: ^/personal/$who/$target" + if [ -n "$email" ]; then + notify_user "$who" "$target" "$email" "$name" + fi + else + $do echo " branch personal/$who/$target already exists." + fi +} + +function notify_user() { + local login=$1 + local target=$2 + local email=$3 + local name=$4 + + $do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b rjsparks@nostrum.com <<-EOF + Hi, + $msg + This mail has been automatically generated by the $program script. + + A new SVN branch has been set up for you for IETF datatracker coding, at + ${svn_url_base}/personal/$login/$target + ${rev:+This branch is based on $rev. }You can check it out by doing + svn co ${svn_url_base}/personal/$login/$target + + There's also a database dump available at + https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz -- this dump is served + via CDN, and should hopefully be swifter to download than the alternatives. + + Please read the instructions about sprint coder setup at + https://trac.ietf.org/tools/ietfdb/wiki/SprintCoderSetup + -- both the workflow description and the details of setting up your + environment. + + + Best regards, + + The IETF tools team (via the $program script) + + EOF +} + +function get_sprinters_info() { + local n=$1 + + curl -L -s "https://trac.ietf.org/trac/ietfdb/wiki/IETF${n}SprintSignUp?format=txt" | \ + grep -E "^\|\|" | \ + grep -Ev "^\|\|\s+\|\|\s+" | \ + tail -n +2 | \ + python3 sprintcoders.py | \ + update "$progdir/sprint${n}.txt" + +} + +# ---------------------------------------------------------------------- +trap 'echo "$program($LINENO): Command failed with error code $? ([$$] "$0" $*)"; exit 1' ERR # ---------------------------------------------------------------------- # Option parsing # Options -shortopts=hm:M:nsvV -longopts=help,meeting=,message=,dry-run,sprint,verbose,version +short_options=hm:M:nsvV +long_options=help,meeting=,message=,dry-run,sprint,verbose,version # Default values num="" msg="" do="" -if [ "$(uname)" = "Linux" ]; then - args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@") - if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi - eval set -- "$args" +if [ "$(uname)" = "Linux" ]; then + # shellcheck disable=SC2086 + # shellcheck disable=SC2048 + if ! args=$(getopt -o $short_options --long $long_options -n $program -- $SV $*); then + die "Terminating..." >&2 + exit 1 + fi + # shellcheck disable=SC2086 + eval set -- $args sed="sed -r" else # Darwin, BSDs - args=$(getopt -o$shortopts $SV $*) - if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi + # shellcheck disable=SC2086 + # shellcheck disable=SC2048 + if ! args=$(getopt -o$short_options $SV $*); then + die "Terminating..." >&2 + exit 1 + fi + # shellcheck disable=SC2086 set -- $args sed="sed -E" fi -while true ; do +while true; do case "$1" in - -h| --help) usage; exit;; # Show this help, then exit - -m| --meeting) num=$2; shift;; # Specify the IETF meeting number - -M| --message) msg=$2; shift;; # Specify extra message text - -n| --dry-run) do="echo -- ==>";; # Only show what would be done - -s| --sprint) SPRINT=1;; # Make branches for sprint sign-ups - -v| --verbose) VERBOSE=1;; # Be more talkative - -V| --version) version; exit;; # Show program version, then exit - --) shift; break;; - *) die "Internal error, inconsistent option specification: '$1'";; + -h | --help) + usage + exit + ;; # Show this help, then exit + -m | --meeting) + num=$2 + shift + ;; # Specify the IETF meeting number + -M | --message) + msg=$2 + shift + ;; # Specify extra message text + -n | --dry-run) do="echo -- ==>" ;; # Only show what would be done + -s | --sprint) SPRINT=1 ;; # Make branches for sprint sign-ups + -v | --verbose) VERBOSE=1 ;; # Be more talkative + -V | --version) + version + exit + ;; # Show program version, then exit + --) + shift + break + ;; + *) die "Internal error, inconsistent option specification: '$1'" ;; esac shift done @@ -118,7 +240,7 @@ done # The program itself who="" -tag=$(svn log -v https://svn.ietf.org/svn/tools/ietfdb/tags/dev/ --limit 1 | grep '/tags/' | awk '{print $2}') +tag=$(svn log -v ${svn_url_base}/tags/dev/ --limit 1 | grep '/tags/' | awk '{print $2}') source="${tag:1}" target="${tag##*/}" @@ -127,95 +249,34 @@ rev="dev tag $target" [ "$1" ] && who="$1" [ "$2" ] && target="${target%.dev*}-$2" -function mksvndir() { - who=$1 - if [ "$2" ]; then dir=$2; else dir=$who; fi - if ! svn info https://svn.ietf.org/svn/tools/ietfdb/personal/$dir >/dev/null 2>&1 ; then - $do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir" - $do svn mkdir https://svn.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code" - else - echo "Repository area personal/$dir is already in place." - fi -} +if [ -z "${who}${SPRINT}" ]; then die "Specify either individual developer name or --sprint"; fi -# dump="ietf_utf8.sql.gz" -# echo "Copying a database dump to www.ietf.org/lib/dt/sprint/$dump" -# scp /www/tools.ietf.org/tools/$dump ietfa:/a/www/www6s/lib/dt/sprint/ - -if [ -z "$who" -a -z "$SPRINT" ]; then die "Specify either individual developer name or --sprint"; fi - -cd $progdir +cd $progdir || exit if [ "$who" ]; then - mksvndir $who - $do svn cp https://svn.ietf.org/svn/tools/ietfdb/$source https://svn.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target" - echo "New branch: ^/personal/$who/$target" + mksvndir "$who" + mksvntarget "$who" "$target" "$source" fi if [ "$SPRINT" ]; then [ "$msg" ] && msg=" $msg " - [ "$num" ] || num=$( < /www/tools.ietf.org/meta/current-ietf-number.txt) - for n in $(seq $((num-3)) $num); do - trac-admin /www/tools.ietf.org/tools/ietfdb wiki export IETF${n}SprintSignUp \ - | egrep "^\|\|" | tail -n +2 | python -c ' -import sys, re -with open("aliases") as afile: - try: - aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ]) - except ValueError: - sys.stderr.write([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ]) - raise - -for line in sys.stdin: - try: - blank, name, email, rest = line.strip().split("||", 3) - email = email.strip() - except ValueError: - sys.stderr.write(line+"\n") - raise - - login, dummy = re.split("[@.]", email, 1) - if email in aliases: - login = aliases[email] - print "\t".join((login.strip().lower(), email.strip().lower(), name.strip())) ' \ - | update $progdir/sprint$n.txt + [ "$num" ] || num=$(curl -L -s "https://tools.ietf.org/meta/current-ietf-number.txt") + for n in $(seq $((num - 3)) "$num"); do + get_sprinters_info "$n" done - cat $(ls $progdir/sprint*.txt | tail -n 2) $progdir/extras.txt | sed -r -e 's/[ \t]*$//' -e 's/[ \t]+/ /g' | sort | uniq | while read login email name; do - echo "" - echo "$login ($name <$email>):" - mksvndir $login - if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target >/dev/null 2>&1 ; then - $do echo " creating $target branch for $login ($name)." - $do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name" \ - && $do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b rjsparks@nostrum.com <<-EOF - Hi, - $msg - This mail has been automatically generated by the $program script. - - A new SVN branch has been set up for you for IETF datatracker coding, at - https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target - ${rev:+This branch is based on $rev. }You can check it out by doing - svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target - - There's also a new database dump available at - https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz -- this dump is served - via CDN, and should hopefully be swifter to download than the alternatives. - - Please read the instructions about sprint coder setup at - https://trac.tools.ietf.org/tools/ietfdb/wiki/SprintCoderSetup - -- both the workflow description and the details of setting up your - environment. - - - Best regards, - - Henrik (via the $program script) - - EOF - else - $do echo " branch personal/$login/$target already exists." - fi + # shellcheck disable=SC2046 + # shellcheck disable=SC2012 + # shellcheck disable=SC2162 + # sed -E (regexp extended) breaks this usage on MacOS 10.15, so back to regular sed. + cat $(ls $progdir/sprint*.txt | tail -n 2) $progdir/extras.txt | \ + sed -e 's/[ \t]*$//' -e 's/[ \t]+/ /g' | \ + sort | uniq | \ + while read login email name; do + echo "" + echo "$login ($name <$email>):" + mksvndir "$login" + mksvntarget "$login" "$target" "$source" "$email" "$name" done fi diff --git a/bin/sprintcoders.py b/bin/sprintcoders.py new file mode 100644 index 000000000..f86bdb3c4 --- /dev/null +++ b/bin/sprintcoders.py @@ -0,0 +1,21 @@ +import sys, re + +with open("aliases") as afile: + try: + aliases = dict([line.strip().split(None, 1) for line in afile.read().splitlines() if line.strip()]) + except ValueError: + sys.stderr.write([line.strip().split(None, 1) for line in afile.read().splitlines() if line.strip()]) + raise + +for line in sys.stdin: + try: + blank, name, email, rest = line.strip().split("||", 3) + email = email.strip() + except ValueError: + sys.stderr.write(line + "\n") + raise + + login, dummy = re.split("[@.]", email, 1) + if email in aliases: + login = aliases[email] + print("\t".join((login.strip().lower(), email.strip().lower(), name.strip()))) diff --git a/bin/update b/bin/update index 42142c403..bcb6e8b12 100755 --- a/bin/update +++ b/bin/update @@ -90,7 +90,11 @@ RESULT="0" QUIET="" # Based on the sample code in /usr/share/doc/util-linux/examples/parse.bash.gz -GETOPT_RESULT=$(getopt -o bc:ef:hn:o:p:qrvV --long backup,maxchg:,empty,file:,help,maxnew:,maxold:,prefix:,report,quiet,verbose,version -n "$program" -- "$@") +if [ "$(uname)" = "Linux" ]; then + GETOPT_RESULT=$(getopt -o bc:ef:hn:o:p:qrvV --long backup,maxchg:,empty,file:,help,maxnew:,maxold:,prefix:,report,quiet,verbose,version -n "$program" -- "$@") +else + GETOPT_RESULT=$(getopt bc:ef:hn:o:p:qrvV "$@") +fi if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi @@ -121,8 +125,8 @@ if [ $CHG ]; then OLD=$CHG; NEW=$CHG; fi if [ $# -lt 1 ]; then echo -e "$program: Missing output filename\n"; usage; fi origfile=$1 -tempfile=$(tempfile) -difffile=$(tempfile) +tempfile=$(mktemp) +difffile=$(mktemp) if [ -e "$origfile" ]; then cp -p $origfile $tempfile # For ownership and permissions