(1) Do not delete and recreate the chroot dir for every new package

    that is built.  This saves a lot of time, especiall when the
    parallelism (the number of jobs per machine, not the number of
    machines) is low.

    However, the build script only blows away /usr/local and
    /usr/X11R6, so if there is a port that does some nasty things
    outside that area, all bets are off.

(2) Better load balancing.  Now, each machine reports its own
    load in a form of a text file, which the master merely aggregates
    to pick the lowest-loaded machine(s).  Other than generally
    running faster (and more up-to-date) under loaded conditions, the
    master script will no longer hold up until a timeout when a
    machine goes down.

5 files changed, 113 insertions, 26 deletions

diff --git a/Tools/portbuild/README b/Tools/portbuild/README
index 73d057f94d14..9ec87e3d3bfb 100644
--- a/Tools/portbuild/README
+++ b/Tools/portbuild/README
@@ -55,11 +55,14 @@ find.
      directories on the build machines.  Create a directory
      ${branch}/chroot on the build machines.
 
- (9) Create the directory ${buildroot}/distfiles, ${branch}/logs and
-     ${branch}/packages/All on the master.  Copy the XFree86 and Motif
-     tarballs to the latter directory.
-
-(10) Run the "checkmachines" script on the master in the background.
+ (9) Create the directory ${buildroot}/distfiles, ${branch}/{errors,logs}
+     and ${branch}/packages/All on the master.  Copy the XFree86 and
+     Motif tarballs to the latter directory.
+
+(10) Run the "reportload" script on the build machines in the
+     background (you can put reportload.sh in /usr/local/etc/rc.d to
+     run it automatically upon reboot), and then run the
+     "checkmachines" script on the master, again in the background.
      This will check all the machines listed in "mlist" periodically
      and print the list of least-loaded machines to "ulist".
 
@@ -81,3 +84,8 @@ find.
 
 (14) Go to ${branch}/packages/All on the master and type "make -k
      -j<whatever> -f ../../Makefile &".  Then wait. :)
+
+(15) Note that the new scheme will not clean up the chroot dirs under
+     ${buildroot}/${branch}/chroot on the build machines, but will
+     instead reuse them during the course of the build.  Make sure you
+     delete them all when your bindist.tar changes.
diff --git a/Tools/portbuild/scripts/checkmachines b/Tools/portbuild/scripts/checkmachines
index e476778d4ce1..127086f7fcc0 100755
--- a/Tools/portbuild/scripts/checkmachines
+++ b/Tools/portbuild/scripts/checkmachines
@@ -2,16 +2,25 @@
 
 buildroot=/a/asami/portbuild
 mlist=${buildroot}/mlist
+stamp=${buildroot}/loads/.stamp
 
 unset DISPLAY
 
 while true; do
+  touch ${stamp}
+  sleep 15
   min=99
   set $(cat $mlist)
   while [ $# -gt 1 ]; do
     m=$1
     l=$2
-    num=$((($(echo $(ssh -n $m ls -1 ${buildroot}/*/chroot | wc -l)) - 3) * 10 / $l))
+    if [ -f ${buildroot}/loads/$m -a \
+      ! -z "$(find ${buildroot}/loads/$m -newer ${stamp})" ]; then
+      num=$(cat ${buildroot}/loads/$m)
+    else
+      num=99
+    fi
+    num=$(($num / $l))
     if [ $num -lt $min ]; then
       mach=$m
       min=$num
@@ -21,5 +30,4 @@ while true; do
     shift 2
   done
   echo "$mach" > ${buildroot}/ulist
-  sleep 5
 done
diff --git a/Tools/portbuild/scripts/portbuild b/Tools/portbuild/scripts/portbuild
index c87d0a7192cc..a26fcfbde6bf 100755
--- a/Tools/portbuild/scripts/portbuild
+++ b/Tools/portbuild/scripts/portbuild
@@ -19,6 +19,9 @@ export DEPENDS_TARGET=/usr/bin/true
 unset MAKEFLAGS
 unset PORTSDIR
 
+# directories to clean
+cleandirs="/usr/local /usr/X11R6"
+
 # 15 minutes
 export FTP_TIMEOUT=900
 export HTTP_TIMEOUT=900
@@ -57,30 +60,71 @@ shift 2
 
 echo "building $pkgname"
 
-chroot=${buildroot}/${branch}/chroot/${pkgname}
+chrootdir=${buildroot}/${branch}/chroot
 bakdir=${buildroot}/${branch}/tarballs
 bindist=${bakdir}/bindist.tar
 packages=${buildroot}/${branch}/packages
 
-rm -rf ${chroot}
-mkdir -p ${chroot}
-tar -C ${chroot} -xf ${bindist}
+found=0
+for dir in ${chrootdir}/*; do
+  if [ -d ${dir}/tmp -a ! -e ${dir}/used ]; then
+    mkdir ${dir}/used 2>/dev/null
+    touch ${dir}/used/${pkgname}
+    if [ $(echo $(echo ${dir}/used/* | wc -w)) = 1 ]; then
+      found=1
+      chroot=${dir}
+      break
+    else
+      rm ${dir}/used/${pkgname}
+      rmdir ${dir}/used 2>/dev/null
+    fi
+  fi
+done
+
+if [ ${found} != 1 ]; then
+
+  chroot=${chrootdir}/$$
+  while [ -d ${chroot} ]; do
+    chroot=${chroot}-
+  done
+
+  mkdir -p ${chroot}/used
+  touch ${chroot}/used/${pkgname}
+  tar -C ${chroot} -xf ${bindist}
+
+  mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.root.dist -p ${chroot} \
+    >/dev/null 2>&1
+  mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.var.dist -p ${chroot}/var \
+    >/dev/null 2>&1
+  mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.usr.dist -p ${chroot}/usr \
+    >/dev/null 2>&1
 
-#cd ${chroot}/usr
-#portcheckout $pkgname | grep '^cvs co' | sh || exit 1
+fi
+
+rm -rf ${chroot}/tmp/*
+cd ${chroot}/tmp
+mkdir -p depends distfiles packages
+
+echo "building ${pkgname} on $(hostname -s)" | tee ${chroot}/tmp/${pkgname}.log
+echo "in directory ${chroot}" | tee -a ${chroot}/tmp/${pkgname}.log
+echo "with arguments: ${args}" | tee -a ${chroot}/tmp/${pkgname}.log
 
 mount -o -2 -r ${master}:${buildroot}/usr/ports ${chroot}/usr/ports
 mount -o -2 -r ${master}:${buildroot}/${branch}/src ${chroot}/usr/src
 mount -o -2 -r ${master}:${buildroot}/usr/opt/doc ${chroot}/usr/opt/doc
 
-mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.root.dist -p ${chroot} >/dev/null 2>&1
-mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.var.dist -p ${chroot}/var >/dev/null 2>&1
-mtree -deU -f ${chroot}/usr/src/etc/mtree/BSD.usr.dist -p ${chroot}/usr >/dev/null 2>&1
+# just in case...
+for dir in ${cleandirs}; do
+  if ! rm -rf ${chroot}${dir} >/dev/null 2>&1; then
+    chflags -R noschg ${chroot}${dir}
+    rm -rf ${chroot}${dir} >/dev/null 2>&1
+  fi
+done
 
 while [ $# -gt 0 ]; do
   if ssh -a ${master} [ -f ${packages}/All/$1 ]; then
     if [ ! -f ${chroot}/tmp/depends/$1 ]; then
-      echo "copying package $1"
+      echo "copying package $1 for ${pkgname}"
       if [ -f ${bakdir}/$1 ]; then
 	cp -p ${bakdir}/$1 ${chroot}/tmp/depends
       else
@@ -95,20 +139,20 @@ scp -p ${master}:${buildroot}/scripts/buildscript ${chroot}
 
 #mount_procfs procfs ${chroot}/proc
 
-echo "${pkgname} built on $(hostname -s)" > ${chroot}/tmp/${pkgname}.log
-echo "with arguments: ${args}" >> ${chroot}/tmp/${pkgname}.log
 chroot ${chroot} /buildscript ${dirname} 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log
 error=$(cat ${chroot}/tmp/status)
 
+scp ${chroot}/tmp/${pkgname}.log ${master}:${buildroot}/${branch}/logs/${pkgname}.log
+
 if [ "${error}" = 0 ]; then
   tar -C ${chroot}/tmp -cf - distfiles | \
     ssh -a $master tar --unlink -C ${buildroot} -xvf -
   tar -C ${chroot}/tmp -cf - packages | \
     ssh -a $master tar --unlink -C ${buildroot}/${branch} -xvf -
   ssh -a $master touch ${buildroot}/${branch}/packages/All/${pkgname}.tgz
-  ssh $master rm -f ${buildroot}/${branch}/logs/${pkgname}.log
+  ssh $master rm -f ${buildroot}/${branch}/errors/${pkgname}.log
 else
-  scp ${chroot}/tmp/${pkgname}.log ${master}:${buildroot}/${branch}/logs/${pkgname}.log
+  scp ${chroot}/tmp/${pkgname}.log ${master}:${buildroot}/${branch}/errors/${pkgname}.log
 fi
 
 #umount ${chroot}/proc
@@ -117,11 +161,17 @@ umount -f ${chroot}/usr/ports
 umount -f ${chroot}/usr/opt/doc
 umount -f ${chroot}/usr/src
 
-if [ $noclean != 1 ]; then
-  if ! rm -rf ${chroot} >/dev/null 2>&1; then
-    chflags -R noschg ${chroot}
-    rm -rf ${chroot} >/dev/null 2>&1
-  fi
+if [ $noclean = 0 -o $error = 0 ]; then
+  rm -rf ${chroot}/tmp/*
+  for dir in ${cleandirs}; do
+    if ! rm -rf ${chroot}${dir} >/dev/null 2>&1; then
+      chflags -R noschg ${chroot}${dir}
+      rm -rf ${chroot}${dir} >/dev/null 2>&1
+    fi
+  done
+  chroot ${chroot} /sbin/ldconfig -R
+  rm ${chroot}/used/${pkgname}
+  rmdir ${chroot}/used
 fi
 
 echo -n "$pkgname done on $(hostname -s) at "
diff --git a/Tools/portbuild/scripts/reportload b/Tools/portbuild/scripts/reportload
new file mode 100755
index 000000000000..a79e73cd09ec
--- /dev/null
+++ b/Tools/portbuild/scripts/reportload
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+me=$(hostname -s)
+master=bento
+buildroot=/a/asami/portbuild
+
+while true; do
+  num=$(echo $(ls -1d ${buildroot}/*/chroot/*/used 2>/dev/null| wc -l))
+  echo "$num" > /tmp/${me}
+  /usr/local/bin/scp -q /tmp/${me} $master:${buildroot}/loads/
+  rm -f /tmp/${me}
+  sleep 5
+done
diff --git a/Tools/portbuild/scripts/reportload.sh b/Tools/portbuild/scripts/reportload.sh
new file mode 100755
index 000000000000..ebae31437c6e
--- /dev/null
+++ b/Tools/portbuild/scripts/reportload.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+s=/a/asami/portbuild/scripts/reportload
+
+if [ -x $s ]; then
+  $s &
+  echo -n ' reportload'
+fi