#!/usr/bin/perl --
# -*- Mode: perl; indent-tabs-mode: nil -*-
#

# This script does not need to run under taint perl because it should
# never be run by the web server. It should be run only by the build
# user id which should have no priveldges.

# A generic build script for tinderbox.  This script will run any
# sequence of shell commands for building or testing an application.
# Users create a configuration file which defines the steps needed to
# build, the build environmental variables and other information which
# tinderbox will need.


# The contents of this file are subject to the Mozilla Public
# License Version 1.1 (the "License"); you may not use this file
# except in compliance with the License. You may obtain a copy of
# the License at http://www.mozilla.org/NPL/
#
# Software distributed under the License is distributed on an "AS
# IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
# implied. See the License for the specific language governing
# rights and limitations under the License.
#
# The Original Code is the Tinderbox build tool.
#
# The Initial Developer of the Original Code is Netscape Communications
# Corporation. Portions created by Netscape are
# Copyright (C) 1998 Netscape Communications Corporation. All
# Rights Reserved.
#

# complete rewrite by Ken Estes for contact info see the
#     mozilla/webtools/tinderbox2/Contact file.
# Contributor(s): 

# $Revision: 1.21 $
# $Date: 2004/01/14 00:22:50 $
# $Author: kestes%walrus.com $
# $Name:  $


# load standard perl libraries

use File::Basename;
use Getopt::Long;
use Sys::Hostname;
use Symbol;
use POSIX;
use IPC::Open3;


# Tinderbox libraries

#  The build script does not use the normal tinderbox libraries, it
#  does need its own local configuration file though.  We find that
#  file via the normal library search path.

use lib '/usr/share/tinderbox/lib', '/etc/tinderbox';


sub usage {

  my ($usage) = <<"EOF";

	$PROGRAM [options]

Global Options:
(These options apply to all builds)

  --daemonize		Run program in the background.
  --once                 Do not loop repeatedly.
  --noreport             Do not mail status reports to the tinderbox server.
  --test                 Print the build commands, do not run the build 
			     commands or mail logs.
  --build BUILDTYPE      The name of the build type to run. You may specify
			 more then one build on the command line by 
			 separating build types with colons (':') each build 
			 will be run in the order given. This will result in 
			 gaps in each of the tinderbox display pages. If you 
			 wish to have multiple builds run at the same time 
			 then run several copies of this program.

  --tree TREENAME        Checkout sources with tree (module and branch)
			 TREENAME. You may specify more then one build on
			 the command line by separating build types with 
			 colons (':') each build will be run in the order
			 given. This will result in gaps in each of the
			 tinderbox display pages. If you wish to have 
			 multiple builds run at the same time then run 
			 several copies of this program.

  --buildcf FILENAME     Provide a config file instead of the default: '$BUILDCF'.
  --mailer MAILER        Specify a different binary to use as the mailing 
                             program.  The default is: '$MAILER'. 

    			In some installations instead of running mail
    			one might prefere to run the build processor
    			directly or use the HTTPPost to send the mail
    			without using a mailer.

		        $MAILER = '/export/home/tinderbox2/bin/processmail_builds';

			$MAILER = 'HTTPPost http://localhost/cgi-bin/cgiwrap/tbox/processmail_builds.cgi';


  --version              Print the version number (same as cvs revision).
  --help                 view this help screen


Build Options:
(These depend on how you configure your builds and may not apply to 
   your configuration.  They are listed here as a suggestion only.)

  --depend               Build depend (must have this option or clobber).
  --clobber              Build clobber (the opposite of depend).
  --tag TREETAG          Checkout CVS sources by tag (-r TREETAG).

Summary:

This program is an automated build client for the tinderbox server.
Builds run continually and the build log file is mailed to tinderbox
for analysis and display.  This script assumes that the hard work in
building the application is taken care of by developer supplied
scripts (shell/Make).  We are assuming that the effort involved in
performing a build is similar to:

	builddir=/tmp/build
	rm -rf \$builddir
	mkdir \$builddir
	cd \$builddir
	cvs checkout Project
	cd \$project
	./configure
	make all
	make test

A 'build' is a list (straight line) of commands run in sequence.
Different lists of commands will have different names.  Notice that
there is a clear separation between the commands used to build the
software and the developer provided scripts which perform the build
(Makefiles, configuration scripts, developer provided build scripts).
Developers are responsible for ensuring that they keep their project
related build scripts up to date so that the build interface does not
change too rapidly.  Then for a given project the build should always
be performed using the same commands and any complex build processing
should be performed inside developer maintained scripts (shell, perl,
Makefile).

Different sequences of build instructions have different 'build
names'.  How you name your builds depends on the type of build testing
you are performing.  You may have architecture build names:
(Macintosh, Solaris, NT) or project/subsystem specific build names
(Sendmail, SeaMonkey, Apache) or functionality specific build names
(Construct, Lint_Test, Functionality_Test, Coverage_Test,
Performance_Test) or some combination of the above.

What make this a bit complicated is that several different parameters
(clobber/dep, the branch name to checkout from version control,
optimize/debug compilation flags, other application specific compile
time flags) will likely be tested by the same build script so our
notion of build must become flexible and encompass families of builds.
This build script will preserve any arguments on the command line
after the string '--' for use by the user supplied program which
defines the builds.

It is assumed that each line in the build script can be safely
executed in its own shell and that if any command fails it will exit
with a non zero exit code.  This is similar to the behavior of
make. Failed builds are started again from the beginning during the
next build cycle.

The build script can be examined by running this program with the
'--test' argument.  This will ensure that no commands are actually run
but the list of commands that would be run when the '--test' argument
is removed is sent to STDOUT for examination.

Before the logs can be mailed to tinderbox there must be information
describing how to display this information.  Tinderbox maintains two
variables for this purpose.  The tinderbox tree name picks which web
page the information will be displayed on.  The tinderbox build name
is the name used for the column that the data will be displayed in.
Typically there is one daemon running continually for each column of
each page. It may be clearer to have different build names display on
the tinderbox columns then are used by this build script.

Users should provide functions which prepare the tinderbox tree and
build name based on both the command line arguments and the
environment.  These functions will need to return strings which are
meaningful and informative for your local environment.

EXAMPLES

/build_shellscript	--buildcf ./generic.sample.buildcf	\
				--build construct --test 	\
				--tree seamonkey

EOF
  ;

print $usage;
exit 0;

}


sub set_static_vars {

  # This functions sets all the static variables which are often
  # configuration parameters.  Since it only sets variables to static
  # quantities it can not fail at run time. Some of these variables are
  # adjusted by parse_args() but aside from that none of these
  # variables are ever written to. All global variables are defined here
  # so we have a list of them and a comment of what they are for.
  
  # Maximum length of a mail line imposed by your mailer.

  $MAX_MAIL_LINE_LEN = 1000;
  
  # These are useful mailers for testing

  # $MAILER = '/bin/true';
  # $MAILER = 'cat > /tmp/mail.out';
  # $MAILER = '/export/home/tinderbox2/bin/processmail_builds';
  # $MAILER = 'HTTPPost http://localhost/cgi-bin/cgiwrap/tbox/processmail_builds.cgi';

  # Usually we send mail to the tinderbox webserver but it is possible
  # to deliver mail directly if the build is running on the same
  # machine as the server.

  $MAILER = '/usr/lib/sendmail -oi -t';

  $ERROR_LOG= '/var/log/tinderbox/build.log';

  $BUILDCF = '/etc/tinderbox/buildcf';

  # minimum time between the start of consecutive builds.  Tinderbox
  # requires this to be greater then 5 minutes.

  $MINIMUM_BUILD_SECONDS = 8*60;

  # The maximum amount of time between tinderbox reports.  These can
  # be as frequent as you like but more frequent will chew up your
  # disk space, network bandwidth and processor time.  People do not
  # need to know exactly where tinderbox is in a build, it only makes
  # sense to make this about the same size as the tinderbox refresh
  # rate.

  $REPORT_TINDERBOX_EVERY_SECONDS = 5*60;

  @ORIG_ARGV = @ARGV;

  # accept the path from our environment.
  
  #  $ENV{'PATH'}= (
  #                 '/bin'.
  #                 ':/usr/bin'.
  #                 ':/usr/local/bin'.
  #                 
  #                 ':/opt/gnu/bin'.
  #                 ':/usr/ucb'.
  #                 ':/usr/ccs/bin'.
  #                 '');
  
  # taint perl requires we clean up these bad environmental variables
  # and it does not hurt to do it here.

  delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV', 'LD_PRELOAD'};

  # sudo deletes these variables as well

  delete @ENV{'KRB_CONF', 'KRB5_CONFIG'};
  delete @ENV{'LOCALDOMAIN', 'RES_OPTIONS', 'HOSTALIASES'};

  # the version number of this tinderbox release.

  $VERSION = '0.09';

  return 1;
}


sub get_env {

# this function sets variables similar to set_static variables.  This
# function may fail only if the OS is in a very strange state.  after
# we leave this function we should be all set up to give good error
# handling, should things fail.

# This function should run as early as possible (directly after
# set_static_vars) so that the error environment is setup in case of
# problems.

  umask 0022; 
  $| = 1;
  $PROGRAM = File::Basename::basename($0);
  $PID = $$; 
  $TIME = time();
  $LOCALTIME = localtime($main::TIME);

  $OS = $^O;
  $HOSTNAME = Sys::Hostname::hostname();

  open (LOG , ">>$ERROR_LOG") ||
    die("Could not open logfile: $ERROR_LOG\n");

  my($old_fh) = select(LOG);
  $| = 1;
  select($old_fh);

  $SIG{'__DIE__'} = \&fatal_error;
  $SIG{'__WARN__'} = \&log_warning;

  return 1;
}



sub fatal_error {
  my  @error = @_;
  foreach $_ (@error) {
    print LOG "[$LOCALTIME] $_";
  }
  print LOG "\n";

  die("\n");
}


sub log_warning {
  my  @error = @_;

  foreach $_ (@error) {
    print LOG "[$LOCALTIME] $_";
  }

  return ;
}


# daemonize a process taken from "Advanced Programming in the UNIX
# Environment" by W. Richard Stevens.


sub daemonize {

    my $pid = fork();

    defined($pid) ||
        die("Could not fork. $!\n");

    # parent stops.
    ($pid == 0) ||
        exit 0;

    # child continues
    chdir("/") || 
        die("Could not change to directory '/'. $!\n");

    setsid;
    umask 0;

    if ($PIDFILE) {
        open(PIDFILE, '>', $PIDFILE) or die "Can't open $PIDFILE: $!";
        print PIDFILE "$$\n";
        close(PIDFILE);
    } else {
        print "$PROGRAM pid: $$\n";
    }

    # Stevens did not mention this but it is now fashionable to close
    # the standard file discriptors.

    # however when I do this users complain about not getting output from
    # their build processes. I need to investigate some other time.

    #close (STDIN) || die ("Could not close STDIN. $!\n");
    #close (STDOUT) || die ("Could not close STDOUT. $!\n");
    #close (STDERR) || die ("Could not close STDERR. $!\n");
    
    return ;
}

sub parse_args {

    %option_linkage = (
		       "version" => \$version,
		       "help" => \$help,
		       "build" => \$BUILD_TYPE,
		       "tree" => \$TREE_NAME,
		       "test" => \$TEST,
		       "mail-each-phase" => \$MAIL_EACH_PHASE,
		       "noreport" => \$NO_REPORT,
		       "mailer" => \$MAILER,
		       "buildcf" => \$BUILDCF,
		       "once" => \$BUILD_ONCE,
                       "daemonize" => \$DAEMONIZE,
                       "pidfile" => \$PIDFILE
		      );
    
    Getopt::Long::config('require_order', 'auto_abbrev', 'ignore_case');

    if( !GetOptions (\%option_linkage,
		     "version", "help", 
		     "build=s", "tree=s", 
                     "test!", "once!", "daemonize!",
		     "mail-each-phase!", "noreport!", 
                     "mailer=s", "buildcf=s", "pidfile=s"
		    ) ) {

      print("Illegal options in \@ARGV: '@ARGV'\n");
      usage();
      exit 1 ;
    }

    if($version) {
      print "Version: $VERSION\n";
      exit 0;  
    }
    
    if ($help) {
      usage();
    }
    
    ($TEST) &&
      ($BUILD_ONCE = 1);

    (@CMD_LOG) =();

    return 1;  
} # parse_args


sub set_server_args {
    my (%args ) = @_;


  # need to load UserDef from buildcf

  UserDef::build_environment(%args);
  # how to build
  $BUILDS = UserDef::build_scripts(%args);
  # which tinderbox web page gets the results
  $SERVER_TREE_NAME = UserDef::tree_name(%args);
  # which column on the page this build gets displayed in
  $SERVER_BUILD_NAME = UserDef::build_name(%args);
  # what regexp the server should use to parse the build log we generate
  $SERVER_ERROR_PARSER = UserDef::error_parser_name(%args);
  

  # person responsible for the build machines.
  $ADMINISTRATOR = UserDef::build_administrator(%args);

  $MAIL_FROM = UserDef::mail_from(%args);
  $MAIL_TO = UserDef::mail_to(%args);
  $MAILER =  UserDef::mailer(%args) || $MAILER;

  check_builds();

  ($SERVER_TREE_NAME) ||
    die("Must define a tree name.\n");
  
  ($SERVER_BUILD_NAME) ||
    die("Must define a build name.\n");

  ($SERVER_ERROR_PARSER) ||
    die("Must define an error parser.\n");
  
  ($ADMINISTRATOR) ||
    die("Must define a build administrator.\n");

  ($MAIL_TO) ||
    die("Must define a mailing address to send builds to.\n");

  ($MAIL_FROM) ||
    die("Must define an account to have the builds be sent from.\n");


  return 1;  
}


# Ensure that user defined build structure has all the mandatory
# parts.

sub check_builds {

  foreach $build_name (sort keys %BUILDS) {
    foreach $i (0..$#{ $BUILDS{$build_name} }) {

      $phase = $BUILDS{$build_name}[$i];
      my ($dir) = $phase->{'dir'};
      my ($error_status) = $phase->{'error_status'};
      my ($phase_name) = $phase->{'phase_name'};
      
      ($phase_name) ||
	die("phase_name not defined ".
	    "in build: $build_name, phase: $i\n");
      
      ($dir) ||
	die("dir not defined ".
	    "in build: $build_name, phase: $phase_name,\n");

      ($dir =~ m!^/!) ||
	die("dir does not specify an absolute path ".
	    "in build: $build_name, phase: $phase_name,\n");

      ($error_status) ||
	die("error_status not defined ".
	    "in build: $build_name, phase: $phase_name,\n");

      ($error_status eq 'success') ||
	die("error_status can not be set to 'success' ".
	    "in build: $build_name, phase: $phase_name,\n");

    }
  }

  return 1;
}

# Send mail to the tinderbox server notifying it of a completed build
# sequence.

sub mail_tinderbox_cmdlog {
  my ($build_status) = @_;    

  $TIME = time();
  $LOCAL_TIME = localtime($TIME);
  
  # Do not have blank lines before the mail headers, 
  # postfix will choke.

  my ($message) = <<"EOF";
From: $MAIL_FROM
To: $MAIL_TO
Subject: Tinderbox Build Update 


tinderbox: tree: $SERVER_TREE_NAME
tinderbox: buildname: $SERVER_BUILD_NAME 
tinderbox: starttime: $START_TIME
tinderbox: localstarttime: $LOCAL_START_TIME
tinderbox: timenow: $TIME
tinderbox: localtimenow: $LOCAL_TIME
tinderbox: errorparser: $SERVER_ERROR_PARSER
tinderbox: status: $build_status
tinderbox: administrator: $ADMINISTRATOR
tinderbox: END


@CMD_LOG


EOF
  ;

  # Split lines into less than 1000 char/line, many
  # mail systems (sendmail) require this.

  $message =~ s/([^\n]{$MAX_MAIL_LINE_LEN})/$1\n/g;    

  # If we are testing then @CMD_LOG contains the list of commands we
  # would have executed. This is where they get printed. We do not
  # need to mail the tinderbox server because the log is really
  # empty.

  if ( ($NO_REPORT) || ($TEST) ){
    print $message;
    return 1;
  }	
  
  my ($pid) = open(MAIL, "|-");
  
  # did we fork a new process?
  
  defined ($pid) || 
    die("Could not fork for cmd: '$MAILER': $!\n");
  
  # If we are the child exec. 
  # Remember the exec function returns only if there is an error.
  
  ($pid) ||
    exec($MAILER) || 
      die("Could not exec: '$MAILER': $!\n");

  print MAIL $message;

  close(MAIL) or
    ($?) or
      die("Could not close '$MAILER': $!\n");
    
  ($?) &&
    die("Could not close '$MAILER' wait_status: $? : $!\n");

  return 1;
}


sub nonblock {
  
  # unbuffer a fh so we can select on it
  
  my ($fh) = shift;
  my $rc = '';
  my $flags = '';
  
  $flags = fcntl($fh, F_GETFL, 0) ||
    die("Could not get flags of socket: $fh : $!\n");
  
  $flags |= O_NONBLOCK;

  $rc = fcntl($fh, F_SETFL, $flags) ||
    die("Could not set flags of socket: $fh : $!\n");
  
  return 1;
}


sub system3 {

# Launch a new child and wait for it to die.  This is like a call to
# system but we get the stdout and stderr in addition to $?.

# call the function like this 

# my  ($wait_status, $log_out, $log_err) = 
# open3(
#     'cmd_vec' => [],
#      );

# cmd_vec is a command to run in execv format.  It is a list not a
# string since we want the safe version of exec

# the system3 function returns:

# wait_status: the wait_status of the child process

# log_out: the stdout that the child process wrote.

# log_err: the stderr the child process wrote.

  my (%args) = @_;

  my $info =">> @{ $args{'cmd_vec'} }\n";
  
#  append_buildlog($info, '');

  # start the process
  
  $fh_in  = gensym(); 
  $fh_out = gensym(); 
  $fh_err = gensym(); 

  ($fh_in && $fh_out && $fh_err) || 
    die ("Could not create new symbol, 'gensym()' object.\n");
  
  my $child_pid = IPC::Open3::open3(
				    $fh_in,
				    $fh_out,
				    $fh_err,
				    @{$args{'cmd_vec'}}
				    );
  
  # this check should be redundant but better safe than sorry
  ($child_pid) || 
      die ("Open3() did not start: \'@{$args{cmd_vec}}\'. $!\n");
  
  close($fh_in) || 
    die("Could not close child stdin: $!\n");
  
  main::nonblock($fh_out);
  main::nonblock($fh_err);
  
  my $reaped_pid = 0;
  my $wait_status = 0;

  # wait for child to die, but keep clearing out stdout and stderr
  # buffers for process so we do not deadlock.  

  while ($reaped_pid != $child_pid) {

    sleep 1;

    $SYSTEM3_LOOP_COUNT++;

    my $new_out = '';
    my $new_err = ''; 

    my $rc = '';
    
    $reaped_pid = waitpid(-1, POSIX::WNOHANG);
    
    if ($reaped_pid == $child_pid) {
      
      # children sometimes signal when they are not dead.
      # we do not care about that.
      
      $wait_status = $?;
      if (!(POSIX::WIFEXITED($wait_status))) {
	$reaped_pid = -1;
      }
      
    }
    
    # do the reading after reaping so we are sure that we exit the
    # loop only after draining the sockets.
    
    # I do not think we need to log $rc errors as they happen
    # frequently and nothing seems wrong:
    #      Resource temporarily unavailable file_handle
    
    do {
      my $data_out = '';
      $rc = sysread($fh_out, $data_out, POSIX::BUFSIZ, 0);
      $new_out .= $data_out;
    } until ($rc <= 0);
    
    do {
      my $data_err = '';
      $rc = sysread($fh_err, $data_err, POSIX::BUFSIZ, 0);
      $new_err .= $data_err;
    } until ($rc <= 0);

    push @CMD_LOG, $new_out.$new_err;

    if ($SYSTEM3_LOOP_COUNT > $REPORT_TINDERBOX_EVERY_SECONDS) {
      $SYSTEM3_LOOP_COUNT = 0;
      mail_tinderbox_cmdlog('building');
    }

  } # while pid
  
  ($reaped_pid != $child_pid) && 
    warn("No Child pid received. ".
	 "reaped_pid: $reaped_pid, ".
	 "child_pid: $child_pid, ".
	 "wait_status: $wait_status, ".
	 "cmd: @{$args{'cmd_vec'}}\n");
  
  # The reads are at the bottom of the loop so we do not need to do
  # any more reading of the filehandles.

  close($fh_out) || 
      &$log_error("Could not close child stdout: $!\n");    
  
  close($fh_err) || 
      &$log_error("Could not close child stderr: $!\n");
  
  return ($wait_status);
} # system3



sub safe_run_shell_commands {
  my ($phase) = @_;

    # catch problems within run_shell_commands and send them to the
    # tinderbox server.  This catches problems with the perl commands
    # chdir or system which would otherwise die.

    my $error_status;
    eval {
        # propagate the error so we can catch it in an eval

           local $SIG{'PIPE'}    = sub { die "@_\n"; };
           local $SIG{'__DIE__'} = sub { die "@_\n"; };

           $error_status = run_shell_commands($phase);
    };

    if ( ($@) || (!($error_status)) ) {
        push @CMD_LOG, $@;
        push @CMD_LOG, "\n";
        $error_status = $phase->{'error_status'};
    }

  return $error_status;
}

# Run one phase of the build process and report the status, stop if
# any command fails.

sub run_shell_commands {
  my ($phase) = @_;
  
  my ($dir) = $phase->{'dir'};
  my ($error_status) = $phase->{'error_status'};
  my ($phase_name) = $phase->{'phase_name'};
  
  my $shell_status;
  
  ($TEST) ||
    chdir($dir) || 
      die("Could not chdir to $dir: $!\n");

  push @CMD_LOG, ("\n>--- Starting Phase: $phase_name  ".
		  "(in dir: $dir) ---<\n\n");
  
  foreach $shell_command (@{ $phase->{'script'} }) {
    
    chomp($shell_command);
    $shell_command =~ s/\#.*//;
    
    $shell_command =~ s/\s+/ /g;
    $shell_command =~ s/^\s+//;
    $shell_command =~ s/\s+$//;
    
    ($shell_command =~ m/^\s+$/) && 
      next;
    
    push @CMD_LOG, ("> $shell_command\n");
    
    # If we are testing do not run the commands, just add them to
    # the build log.
    
    ($TEST) && 
      next;

    # I should replace this open() with an accept() so that if the
    # command takes 30 minutes to run we can still send log updates to
    # the tinderbox server every 10 minutes.
    
#    open(CMD, "$shell_command 2>&1 |") or 
#      die("Could not run '$shell_command': $!\n");
    
#    my (@results) = <CMD>;
    
#    push @CMD_LOG, (@results);
    
#    close(CMD) or 
#      ($?) or
#	die("Could not close '$shell_command': $!\n");
    
#    $shell_status = $?;

    $shell_status = system3(
                            'cmd_vec' => [$shell_command],
                            );

    if ($shell_status) {
        return $error_status;
    }

  }
  
  push @CMD_LOG, "\n";

  return 'success';
}



# This function ensures that builds do not start too frequently.  It
# will sleep if and only if the last build finished too quickly.

sub slow_build_speed {
  
  my ($actual_build_seconds) = $END_TIME - $START_TIME;
  
  if ($actual_build_seconds < $MINIMUM_BUILD_SECONDS) {
    my ($sleep_time) = $MINIMUM_BUILD_SECONDS - $actual_build_seconds;
    sleep($sleep_time);
  }

  return 1;
}




#-------------- 
# Main function
#-------------- 

{
  set_static_vars();
  get_env();
  parse_args();

  require $BUILDCF;

  ($DAEMONIZE) &&
      daemonize();    


  do { # foreach build

      foreach $build_type (split(/:/, $BUILD_TYPE)) {

          foreach $tree_name (split(/:/, $TREE_NAME)) {

              # We need to rerun the user code each iteration so that
              # they can set file names with embedded dates.  This
              # allows each build to be saved in a separate tar file
              # like: binaries.20010814.13:42:16.tar.gz

              %args = (
                   argv => [@ARGV],
                   build_type => $build_type,
                   tree_name => $tree_name,
                   
                   hostname => $HOSTNAME,
                   program =>   $PROGRAM, 
                   os =>	$OS,
                   );
          
              set_server_args(%args);
              
              (@CMD_LOG) =();
              
              $START_TIME = time();
              $LOCAL_START_TIME = localtime($START_TIME);
              $SYSTEM3_LOOP_COUNT = 0;
              
              my ($status);
              
              foreach $phase (@{ $BUILDS->{$build_type} }) {
                  
                  ($MAIL_EACH_PHASE) && 
                      mail_tinderbox_cmdlog('building');
                  
                  $status = safe_run_shell_commands($phase);
                  
                  ($status ne 'success') && 
                      last;
                  
              } # foreach $phase
              
              $END_TIME = time();
              $LOCAL_END_TIME = localtime($END_TIME);
              
              mail_tinderbox_cmdlog($status);
              
              ($BUILD_ONCE) ||
                  slow_build_speed();
              
          } # foreach buildtype
          
      } # foreach tree_name 
      
  } while (!($BUILD_ONCE)); # foreach build
  
  exit 0;
}



