altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

Merged in [19071] from mark@painless-security.com:

Browse source 
Update the mkdevbranch script for new Trac location, don't clobber existing paths
The script has been cleaned up, adjusted to avoid clobbering an SVN path that already exists when running it for a single sprinter login, and run with the new locations for the Trac system.
This is a decently huge rewrite of the script.  It's pulled a lot of logic into functions, excised the python code into its own file, etc.
 - Legacy-Id: 19076
Note: SVN reference [19071] has been migrated to Git commit 537b76eb76170d69036999f8a8ef20fd81caca93
This commit is contained in:
Robert Sparks 2021-06-04 18:42:01 +00:00
parent 6b153bcf41
commit ddcd80766e
3 changed files with 198 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

253
bin/mkdevbranch
View file

@ -3,6 +3,7 @@
version=0.24
program=${0##*/}
progdir=${0%/*}
svn_url_base="https://svn.ietf.org/svn/tools/ietfdb"
if [ "$progdir" = "$program" ]; then progdir="."; fi
# ----------------------------------------------------------------------
@ -25,9 +26,13 @@ DESCRIPTION
EOF
echo -e "OPTIONS"
if [ "$(uname)" = "Linux" ]; then
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | tr -s "\t|" "\t," | sed -r -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
# shellcheck disable=SC2086
# shellcheck disable=SC2016
grep -E "^\s+[-][A-Za-z| -]+\*?\)\s+[A-Za-z].+#" "$0" | tr -s "\t|" "\t," | $sed -e 's/\)[ \t]+([A-Z]+)=\$2[^#]*#/=\1\t/' -e 's/\)[^#]*#/\t/'
else
egrep "^[ ]+[-][A-Za-z| -]+\*?\)[ ]+[A-Za-z].+#" $0 | sed 's/\|.*\$2[^#]*#/ /'| sed -E 's/\|.*\)[^#]*#/ /'
# shellcheck disable=SC2086
# shellcheck disable=SC2016
grep -E "^\s+[-][A-Za-z| -]+\*?\)\s+[A-Za-z].+#" "$0" | $sed 's/\|.*\$2[^#]*#/ /' | $sed 's/\|.*\)[^#]*#/ /'
fi
cat <<EOF
@ -57,10 +62,9 @@ function die() {
}
function warn() {
echo "$program: Warning: $*" 1>&2;
echo "$program: Warning: $*" 1>&2
}
function note() {
if [ -n "$VERBOSE" ]; then echo -e "$*"; fi
}
@ -71,15 +75,108 @@ function version() {
}
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR
function check_svn_path_exists() {
local __resultvar=$1
local __path=$2
local myresult
svn info "${__path}" >/dev/null 2>&1
myresult=$?
# shellcheck disable=SC2086
eval $__resultvar="'$myresult'"
}
function mksvndir() {
who=$1
if [ "$2" ]; then dir=$2; else dir=$who; fi
check_svn_path_exists exists "${svn_url_base}/personal/$dir"
# shellcheck disable=SC2154
if [ "$exists" != "0" ]; then
$do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
$do svn mkdir "${svn_url_base}/personal/$dir" -m "Personal SVN dir for $who, for IETF datatracker code"
else
echo "Repository area personal/$dir is already in place."
fi
}
function mksvntarget() {
local who=$1
local target=$2
local source=$3
local email=$4
local name=$5
check_svn_path_exists exists "${svn_url_base}/personal/$who/$target"
if [ "$exists" != "0" ]; then
$do echo " creating $target branch for $who ($name)."
$do svn cp "${svn_url_base}/$source" \
"${svn_url_base}/personal/$who/$target/" \
-m "New branch for $target"
$do echo "New branch: ^/personal/$who/$target"
if [ -n "$email" ]; then
notify_user "$who" "$target" "$email" "$name"
fi
else
$do echo " branch personal/$who/$target already exists."
fi
}
function notify_user() {
local login=$1
local target=$2
local email=$3
local name=$4
$do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b rjsparks@nostrum.com <<-EOF
Hi,
$msg
This mail has been automatically generated by the $program script.
A new SVN branch has been set up for you for IETF datatracker coding, at
${svn_url_base}/personal/$login/$target
${rev:+This branch is based on $rev. }You can check it out by doing
svn co ${svn_url_base}/personal/$login/$target
There's also a database dump available at
https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz -- this dump is served
via CDN, and should hopefully be swifter to download than the alternatives.
Please read the instructions about sprint coder setup at
https://trac.ietf.org/tools/ietfdb/wiki/SprintCoderSetup
-- both the workflow description and the details of setting up your
environment.
Best regards,
The IETF tools team (via the $program script)
EOF
}
function get_sprinters_info() {
local n=$1
curl -L -s "https://trac.ietf.org/trac/ietfdb/wiki/IETF${n}SprintSignUp?format=txt" | \
grep -E "^\|\|" | \
grep -Ev "^\|\|\s+\|\|\s+" | \
tail -n +2 | \
python3 sprintcoders.py | \
update "$progdir/sprint${n}.txt"
}
# ----------------------------------------------------------------------
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] "$0" $*)"; exit 1' ERR
# ----------------------------------------------------------------------
# Option parsing
# Options
shortopts=hm:M:nsvV
longopts=help,meeting=,message=,dry-run,sprint,verbose,version
short_options=hm:M:nsvV
long_options=help,meeting=,message=,dry-run,sprint,verbose,version
# Default values
num=""
@ -87,28 +184,53 @@ msg=""
do=""
if [ "$(uname)" = "Linux" ]; then
args=$(getopt -o "$shortopts" --long "$longopts" -n '$program' -- $SV "$@")
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
eval set -- "$args"
# shellcheck disable=SC2086
# shellcheck disable=SC2048
if ! args=$(getopt -o $short_options --long $long_options -n $program -- $SV $*); then
die "Terminating..." >&2
exit 1
fi
# shellcheck disable=SC2086
eval set -- $args
sed="sed -r"
else
# Darwin, BSDs
args=$(getopt -o$shortopts $SV $*)
if [ $? != 0 ] ; then die "Terminating..." >&2 ; exit 1 ; fi
# shellcheck disable=SC2086
# shellcheck disable=SC2048
if ! args=$(getopt -o$short_options $SV $*); then
die "Terminating..." >&2
exit 1
fi
# shellcheck disable=SC2086
set -- $args
sed="sed -E"
fi
while true; do
case "$1" in
-h| --help) usage; exit;; # Show this help, then exit
-m| --meeting) num=$2; shift;; # Specify the IETF meeting number
-M| --message) msg=$2; shift;; # Specify extra message text
-h | --help)
usage
exit
;; # Show this help, then exit
-m | --meeting)
num=$2
shift
;; # Specify the IETF meeting number
-M | --message)
msg=$2
shift
;; # Specify extra message text
-n | --dry-run) do="echo -- ==>" ;; # Only show what would be done
-s | --sprint) SPRINT=1 ;; # Make branches for sprint sign-ups
-v | --verbose) VERBOSE=1 ;; # Be more talkative
-V| --version) version; exit;; # Show program version, then exit
--) shift; break;;
-V | --version)
version
exit
;; # Show program version, then exit
--)
shift
break
;;
*) die "Internal error, inconsistent option specification: '$1'" ;;
esac
shift
@ -118,7 +240,7 @@ done
# The program itself
who=""
tag=$(svn log -v https://svn.ietf.org/svn/tools/ietfdb/tags/dev/ --limit 1 | grep '/tags/' | awk '{print $2}')
tag=$(svn log -v ${svn_url_base}/tags/dev/ --limit 1 | grep '/tags/' | awk '{print $2}')
source="${tag:1}"
target="${tag##*/}"
@ -127,95 +249,34 @@ rev="dev tag $target"
[ "$1" ] && who="$1"
[ "$2" ] && target="${target%.dev*}-$2"
function mksvndir() {
who=$1
if [ "$2" ]; then dir=$2; else dir=$who; fi
if ! svn info https://svn.ietf.org/svn/tools/ietfdb/personal/$dir >/dev/null 2>&1 ; then
$do echo "Creating personal directory area for IETF datatracker coding: /personal/$dir"
$do svn mkdir https://svn.ietf.org/svn/tools/ietfdb/personal/$dir -m "Personal SVN dir for $who, for IETF datatracker code"
else
echo "Repository area personal/$dir is already in place."
fi
}
if [ -z "${who}${SPRINT}" ]; then die "Specify either individual developer name or --sprint"; fi
# dump="ietf_utf8.sql.gz"
# echo "Copying a database dump to www.ietf.org/lib/dt/sprint/$dump"
# scp /www/tools.ietf.org/tools/$dump ietfa:/a/www/www6s/lib/dt/sprint/
if [ -z "$who" -a -z "$SPRINT" ]; then die "Specify either individual developer name or --sprint"; fi
cd $progdir
cd $progdir || exit
if [ "$who" ]; then
mksvndir $who
$do svn cp https://svn.ietf.org/svn/tools/ietfdb/$source https://svn.ietf.org/svn/tools/ietfdb/personal/$who/$target/ -m "New branch for $target"
echo "New branch: ^/personal/$who/$target"
mksvndir "$who"
mksvntarget "$who" "$target" "$source"
fi
if [ "$SPRINT" ]; then
[ "$msg" ] && msg="
$msg
"
[ "$num" ] || num=$( < /www/tools.ietf.org/meta/current-ietf-number.txt)
for n in $(seq $((num-3)) $num); do
trac-admin /www/tools.ietf.org/tools/ietfdb wiki export IETF${n}SprintSignUp \
| egrep "^\|\|" | tail -n +2 | python -c '
import sys, re
with open("aliases") as afile:
try:
aliases = dict([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
except ValueError:
sys.stderr.write([ line.strip().split(None,1) for line in afile.read().splitlines() if line.strip() ])
raise
for line in sys.stdin:
try:
blank, name, email, rest = line.strip().split("||", 3)
email = email.strip()
except ValueError:
sys.stderr.write(line+"\n")
raise
login, dummy = re.split("[@.]", email, 1)
if email in aliases:
login = aliases[email]
print "\t".join((login.strip().lower(), email.strip().lower(), name.strip())) ' \
| update $progdir/sprint$n.txt
[ "$num" ] || num=$(curl -L -s "https://tools.ietf.org/meta/current-ietf-number.txt")
for n in $(seq $((num - 3)) "$num"); do
get_sprinters_info "$n"
done
cat $(ls $progdir/sprint*.txt | tail -n 2) $progdir/extras.txt | sed -r -e 's/[ \t]*$//' -e 's/[ \t]+/ /g' | sort | uniq | while read login email name; do
# shellcheck disable=SC2046
# shellcheck disable=SC2012
# shellcheck disable=SC2162
# sed -E (regexp extended) breaks this usage on MacOS 10.15, so back to regular sed.
cat $(ls $progdir/sprint*.txt | tail -n 2) $progdir/extras.txt | \
sed -e 's/[ \t]*$//' -e 's/[ \t]+/ /g' | \
sort | uniq | \
while read login email name; do
echo ""
echo "$login ($name <$email>):"
mksvndir $login
if ! svn info https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target >/dev/null 2>&1 ; then
$do echo " creating $target branch for $login ($name)."
$do svn cp https://svn.tools.ietf.org/svn/tools/ietfdb/$source https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target/ -m "New IETF datatracker coding branch for $name" \
&& $do mail "$name <$email>" -s "A new SVN branch for you for IETF datatracker coding${rev:+, based on $rev}." -b rjsparks@nostrum.com <<-EOF
Hi,
$msg
This mail has been automatically generated by the $program script.
A new SVN branch has been set up for you for IETF datatracker coding, at
https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target
${rev:+This branch is based on $rev. }You can check it out by doing
svn co https://svn.tools.ietf.org/svn/tools/ietfdb/personal/$login/$target
There's also a new database dump available at
https://www.ietf.org/lib/dt/sprint/ietf_utf8.sql.gz -- this dump is served
via CDN, and should hopefully be swifter to download than the alternatives.
Please read the instructions about sprint coder setup at
https://trac.tools.ietf.org/tools/ietfdb/wiki/SprintCoderSetup
-- both the workflow description and the details of setting up your
environment.
Best regards,
Henrik (via the $program script)
EOF
else
$do echo " branch personal/$login/$target already exists."
fi
mksvndir "$login"
mksvntarget "$login" "$target" "$source" "$email" "$name"
done
fi

21
bin/sprintcoders.py Normal file
View file

@ -0,0 +1,21 @@
import sys, re
with open("aliases") as afile:
try:
aliases = dict([line.strip().split(None, 1) for line in afile.read().splitlines() if line.strip()])
except ValueError:
sys.stderr.write([line.strip().split(None, 1) for line in afile.read().splitlines() if line.strip()])
raise
for line in sys.stdin:
try:
blank, name, email, rest = line.strip().split("||", 3)
email = email.strip()
except ValueError:
sys.stderr.write(line + "\n")
raise
login, dummy = re.split("[@.]", email, 1)
if email in aliases:
login = aliases[email]
print("\t".join((login.strip().lower(), email.strip().lower(), name.strip())))

8
bin/update
View file

@ -90,7 +90,11 @@ RESULT="0"
QUIET=""
# Based on the sample code in /usr/share/doc/util-linux/examples/parse.bash.gz
if [ "$(uname)" = "Linux" ]; then
GETOPT_RESULT=$(getopt -o bc:ef:hn:o:p:qrvV --long backup,maxchg:,empty,file:,help,maxnew:,maxold:,prefix:,report,quiet,verbose,version -n "$program" -- "$@")
else
GETOPT_RESULT=$(getopt bc:ef:hn:o:p:qrvV "$@")
fi
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
@ -121,8 +125,8 @@ if [ $CHG ]; then OLD=$CHG; NEW=$CHG; fi
if [ $# -lt 1 ]; then echo -e "$program: Missing output filename\n"; usage; fi
origfile=$1
tempfile=$(tempfile)
difffile=$(tempfile)
tempfile=$(mktemp)
difffile=$(mktemp)
if [ -e "$origfile" ]; then
cp -p $origfile $tempfile # For ownership and permissions