]> Git Repo - J-u-boot.git/blobdiff - tools/buildman/builder.py
buildman: Allow adjusting board config on the fly
[J-u-boot.git] / tools / buildman / builder.py
index 784c64122ba67285f2552926527126b52ae6b1b8..720bbb2cf4dc8f4a79237c94c8366c81396f84b8 100644 (file)
@@ -17,13 +17,23 @@ import sys
 import threading
 import time
 
-import builderthread
-import command
-import gitutil
-import terminal
-from terminal import Print
-import toolchain
-
+from buildman import builderthread
+from buildman import toolchain
+from patman import command
+from patman import gitutil
+from patman import terminal
+from patman.terminal import Print
+
+# This indicates an new int or hex Kconfig property with no default
+# It hangs the build since the 'conf' tool cannot proceed without valid input.
+#
+# We get a repeat sequence of something like this:
+# >>
+# Break things (BREAK_ME) [] (NEW)
+# Error in reading or end of file.
+# <<
+# which indicates that BREAK_ME has an empty default
+RE_NO_DEFAULT = re.compile(b'\((\w+)\) \[] \(NEW\)')
 
 """
 Theory of Operation
@@ -71,12 +81,12 @@ As an example, say we are building branch 'us-net' for boards 'sandbox' and
 like this:
 
 us-net/             base directory
-    01_of_02_g4ed4ebc_net--Add-tftp-speed-/
+    01_g4ed4ebc_net--Add-tftp-speed-/
         sandbox/
             u-boot.bin
         seaboard/
             u-boot.bin
-    02_of_02_g4ed4ebc_net--Check-tftp-comp/
+    02_g4ed4ebc_net--Check-tftp-comp/
         sandbox/
             u-boot.bin
         seaboard/
@@ -91,6 +101,15 @@ u-boot/             source directory
     .git/           repository
 """
 
+"""Holds information about a particular error line we are outputing
+
+   char: Character representation: '+': error, '-': fixed error, 'w+': warning,
+       'w-' = fixed warning
+   boards: List of Board objects which have line in the error/warning output
+   errline: The text of the error line
+"""
+ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
+
 # Possible build outcomes
 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 
@@ -154,8 +173,6 @@ class Builder:
         force_build_failures: If a previously-built build (i.e. built on
             a previous run of buildman) is marked as failed, rebuild it.
         git_dir: Git directory containing source repository
-        last_line_len: Length of the last line we printed (used for erasing
-            it with new progress information)
         num_jobs: Number of jobs to run at once (passed to make as -j)
         num_threads: Number of builder threads to run
         out_queue: Queue of results to process
@@ -174,6 +191,9 @@ class Builder:
         in_tree: Build U-Boot in-tree instead of specifying an output
             directory separate from the source code. This option is really
             only useful for testing in-tree builds.
+        work_in_output: Use the output directory as the work directory and
+            don't write to a separate output directory.
+        thread_exceptions: List of exceptions raised by thread jobs
 
     Private members:
         _base_board_dict: Last-summarised Dict of boards
@@ -184,10 +204,15 @@ class Builder:
         _next_delay_update: Next time we plan to display a progress update
                 (datatime)
         _show_unknown: Show unknown boards (those not built) in summary
+        _start_time: Start time for the build
         _timestamps: List of timestamps for the completion of the last
             last _timestamp_count builds. Each is a datetime object.
         _timestamp_count: Number of timestamps to keep in our list.
         _working_dir: Base working directory containing all threads
+        _single_builder: BuilderThread object for the singer builder, if
+            threading is not being used
+        _terminated: Thread was terminated due to an error
+        _restarting_config: True if 'Restart config' is detected in output
     """
     class Outcome:
         """Records a build outcome for a single make invocation
@@ -222,9 +247,10 @@ class Builder:
     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
                  gnu_make='make', checkout=True, show_unknown=True, step=1,
                  no_subdirs=False, full_path=False, verbose_build=False,
-                 incremental=False, per_board_out_dir=False,
+                 mrproper=False, per_board_out_dir=False,
                  config_only=False, squash_config_y=False,
-                 warnings_as_errors=False):
+                 warnings_as_errors=False, work_in_output=False,
+                 test_thread_exceptions=False, adjust_cfg=None):
         """Create a new Builder object
 
         Args:
@@ -243,17 +269,33 @@ class Builder:
             full_path: Return the full path in CROSS_COMPILE and don't set
                 PATH
             verbose_build: Run build with V=1 and don't use 'make -s'
-            incremental: Always perform incremental builds; don't run make
-                mrproper when configuring
+            mrproper: Always run 'make mrproper' when configuring
             per_board_out_dir: Build in a separate persistent directory per
                 board rather than a thread-specific directory
             config_only: Only configure each build, don't build it
             squash_config_y: Convert CONFIG options with the value 'y' to '1'
             warnings_as_errors: Treat all compiler warnings as errors
+            work_in_output: Use the output directory as the work directory and
+                don't write to a separate output directory.
+            test_thread_exceptions: Uses for tests only, True to make the
+                threads raise an exception instead of reporting their result.
+                This simulates a failure in the code somewhere
+            adjust_cfg_list (list of str): List of changes to make to .config
+                file before building. Each is one of (where C is the config
+                option with or without the CONFIG_ prefix)
+
+                    C to enable C
+                    ~C to disable C
+                    C=val to set the value of C (val must have quotes if C is
+                        a string Kconfig
+
         """
         self.toolchains = toolchains
         self.base_dir = base_dir
-        self._working_dir = os.path.join(base_dir, '.bm-work')
+        if work_in_output:
+            self._working_dir = base_dir
+        else:
+            self._working_dir = os.path.join(base_dir, '.bm-work')
         self.threads = []
         self.do_make = self.Make
         self.gnu_make = gnu_make
@@ -268,6 +310,7 @@ class Builder:
         self._build_period_us = None
         self._complete_delay = None
         self._next_delay_update = datetime.now()
+        self._start_time = datetime.now()
         self.force_config_on_failure = True
         self.force_build_failures = False
         self.force_reconfig = False
@@ -280,8 +323,13 @@ class Builder:
         self.config_only = config_only
         self.squash_config_y = squash_config_y
         self.config_filenames = BASE_CONFIG_FILENAMES
+        self.work_in_output = work_in_output
+        self.adjust_cfg = adjust_cfg
+
         if not self.squash_config_y:
             self.config_filenames += EXTRA_CONFIG_FILENAMES
+        self._terminated = False
+        self._restarting_config = False
 
         self.warnings_as_errors = warnings_as_errors
         self.col = terminal.Color()
@@ -291,21 +339,30 @@ class Builder:
         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
         self._re_dtb_warning = re.compile('(.*): Warning .*')
         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
-
-        self.queue = queue.Queue()
-        self.out_queue = queue.Queue()
-        for i in range(self.num_threads):
-            t = builderthread.BuilderThread(self, i, incremental,
-                    per_board_out_dir)
+        self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
+                                                re.MULTILINE | re.DOTALL)
+
+        self.thread_exceptions = []
+        self.test_thread_exceptions = test_thread_exceptions
+        if self.num_threads:
+            self._single_builder = None
+            self.queue = queue.Queue()
+            self.out_queue = queue.Queue()
+            for i in range(self.num_threads):
+                t = builderthread.BuilderThread(
+                        self, i, mrproper, per_board_out_dir,
+                        test_exception=test_thread_exceptions)
+                t.setDaemon(True)
+                t.start()
+                self.threads.append(t)
+
+            t = builderthread.ResultThread(self)
             t.setDaemon(True)
             t.start()
             self.threads.append(t)
-
-        self.last_line_len = 0
-        t = builderthread.ResultThread(self)
-        t.setDaemon(True)
-        t.start()
-        self.threads.append(t)
+        else:
+            self._single_builder = builderthread.BuilderThread(
+                self, -1, mrproper, per_board_out_dir)
 
         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
         self.re_make_err = re.compile('|'.join(ignore_lines))
@@ -324,16 +381,22 @@ class Builder:
     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
                           show_detail=False, show_bloat=False,
                           list_error_boards=False, show_config=False,
-                          show_environment=False):
+                          show_environment=False, filter_dtb_warnings=False,
+                          filter_migration_warnings=False):
         """Setup display options for the builder.
 
-        show_errors: True to show summarised error/warning info
-        show_sizes: Show size deltas
-        show_detail: Show detail for each board
-        show_bloat: Show detail for each function
-        list_error_boards: Show the boards which caused each error/warning
-        show_config: Show config deltas
-        show_environment: Show environment deltas
+        Args:
+            show_errors: True to show summarised error/warning info
+            show_sizes: Show size deltas
+            show_detail: Show size delta detail for each board if show_sizes
+            show_bloat: Show detail for each function
+            list_error_boards: Show the boards which caused each error/warning
+            show_config: Show config deltas
+            show_environment: Show environment deltas
+            filter_dtb_warnings: Filter out any warnings from the device-tree
+                compiler
+            filter_migration_warnings: Filter out any warnings about migrating
+                a board to driver model
         """
         self._show_errors = show_errors
         self._show_sizes = show_sizes
@@ -342,6 +405,8 @@ class Builder:
         self._list_error_boards = list_error_boards
         self._show_config = show_config
         self._show_environment = show_environment
+        self._filter_dtb_warnings = filter_dtb_warnings
+        self._filter_migration_warnings = filter_migration_warnings
 
     def _AddTimestamp(self):
         """Add a new timestamp to the list and record the build period.
@@ -372,22 +437,6 @@ class Builder:
             self._timestamps.popleft()
             count -= 1
 
-    def ClearLine(self, length):
-        """Clear any characters on the current line
-
-        Make way for a new line of length 'length', by outputting enough
-        spaces to clear out the old line. Then remember the new length for
-        next time.
-
-        Args:
-            length: Length of new line, in characters
-        """
-        if length < self.last_line_len:
-            Print(' ' * (self.last_line_len - length), newline=False)
-            Print('\r', newline=False)
-        self.last_line_len = length
-        sys.stdout.flush()
-
     def SelectCommit(self, commit, checkout=True):
         """Checkout the selected commit for this build
         """
@@ -406,9 +455,35 @@ class Builder:
             args: Arguments to pass to make
             kwargs: Arguments to pass to command.RunPipe()
         """
+
+        def check_output(stream, data):
+            if b'Restart config' in data:
+                self._restarting_config = True
+
+            # If we see 'Restart config' following by multiple errors
+            if self._restarting_config:
+                m = RE_NO_DEFAULT.findall(data)
+
+                # Number of occurences of each Kconfig item
+                multiple = [m.count(val) for val in set(m)]
+
+                # If any of them occur more than once, we have a loop
+                if [val for val in multiple if val > 1]:
+                    self._terminated = True
+                    return True
+            return False
+
+        self._restarting_config = False
+        self._terminated  = False
         cmd = [self.gnu_make] + list(args)
         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
-                cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
+                cwd=cwd, raise_on_error=False, infile='/dev/null',
+                output_func=check_output, **kwargs)
+
+        if self._terminated:
+            # Try to be helpful
+            result.stderr += '(** did you define an int/hex Kconfig with no default? **)'
+
         if self.verbose_build:
             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
@@ -433,8 +508,7 @@ class Builder:
             if result.already_done:
                 self.already_done += 1
             if self._verbose:
-                Print('\r', newline=False)
-                self.ClearLine(0)
+                terminal.PrintClear()
                 boards_selected = {target : result.brd}
                 self.ResetResultSummary(boards_selected)
                 self.ProduceResultSummary(result.commit_upto, self.commits,
@@ -448,22 +522,21 @@ class Builder:
         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 
-        name = ' /%-5d  ' % self.count
+        line += ' /%-5d  ' % self.count
+        remaining = self.count - self.upto
+        if remaining:
+            line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
+        else:
+            line += ' ' * 8
 
         # Add our current completion time estimate
         self._AddTimestamp()
         if self._complete_delay:
-            name += '%s  : ' % self._complete_delay
-        # When building all boards for a commit, we can print a commit
-        # progress message.
-        if result and result.commit_upto is None:
-            name += 'commit %2d/%-3d' % (self.commit_upto + 1,
-                    self.commit_count)
-
-        name += target
-        Print(line + name, newline=False)
-        length = 16 + len(name)
-        self.ClearLine(length)
+            line += '%s  : ' % self._complete_delay
+
+        line += target
+        terminal.PrintClear()
+        Print(line, newline=False, limit_to_line=True)
 
     def _GetOutputDir(self, commit_upto):
         """Get the name of the output directory for a commit number
@@ -473,12 +546,16 @@ class Builder:
         Args:
             commit_upto: Commit number to use (0..self.count-1)
         """
+        if self.work_in_output:
+            return self._working_dir
+
         commit_dir = None
         if self.commits:
             commit = self.commits[commit_upto]
             subject = commit.subject.translate(trans_valid_chars)
-            commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
-                    self.commit_count, commit.hash, subject[:20]))
+            # See _GetOutputSpaceRemovals() which parses this name
+            commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
+                    commit.hash, subject[:20]))
         elif not self.no_subdirs:
             commit_dir = 'current'
         if not commit_dir:
@@ -495,6 +572,8 @@ class Builder:
             target: Target name
         """
         output_dir = self._GetOutputDir(commit_upto)
+        if self.work_in_output:
+            return output_dir
         return os.path.join(output_dir, target)
 
     def GetDoneFile(self, commit_upto, target):
@@ -558,9 +637,16 @@ class Builder:
             New list with only interesting lines included
         """
         out_lines = []
+        if self._filter_migration_warnings:
+            text = '\n'.join(lines)
+            text = self._re_migration_warning.sub('', text)
+            lines = text.splitlines()
         for line in lines:
-            if not self.re_make_err.search(line):
-                out_lines.append(line)
+            if self.re_make_err.search(line):
+                continue
+            if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
+                continue
+            out_lines.append(line)
         return out_lines
 
     def ReadFuncSizes(self, fname, fd):
@@ -979,7 +1065,7 @@ class Builder:
 
         The summary takes the form of one line per architecture. The
         line contains deltas for each of the sections (+ means the section
-        got bigger, - means smaller). The nunmbers are the average number
+        got bigger, - means smaller). The numbers are the average number
         of bytes that a board in this section increased by.
 
         For example:
@@ -992,7 +1078,7 @@ class Builder:
                 board.target
             board_dict: Dict containing boards for which we built this
                 commit, keyed by board.target. The value is an Outcome object.
-            show_detail: Show detail for each board
+            show_detail: Show size delta detail for each board
             show_bloat: Show detail for each function
         """
         arch_list = {}
@@ -1109,7 +1195,7 @@ class Builder:
             environment: Dictionary keyed by environment variable, Each
                      value is the value of environment variable.
             show_sizes: Show image size deltas
-            show_detail: Show detail for each board
+            show_detail: Show size delta detail for each board if show_sizes
             show_bloat: Show detail for each function
             show_config: Show config changes
             show_environment: Show environment changes
@@ -1119,32 +1205,52 @@ class Builder:
 
             Args:
                 line: Error line to search for
+                line_boards: boards to search, each a Board
             Return:
-                String containing a list of boards with that error line, or
-                '' if the user has not requested such a list
+                List of boards with that error line, or [] if the user has not
+                    requested such a list
             """
+            boards = []
+            board_set = set()
             if self._list_error_boards:
-                names = []
                 for board in line_boards[line]:
-                    if not board.target in names:
-                        names.append(board.target)
-                names_str = '(%s) ' % ','.join(names)
-            else:
-                names_str = ''
-            return names_str
+                    if not board in board_set:
+                        boards.append(board)
+                        board_set.add(board)
+            return boards
 
         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
                             char):
+            """Calculate the required output based on changes in errors
+
+            Args:
+                base_lines: List of errors/warnings for previous commit
+                base_line_boards: Dict keyed by error line, containing a list
+                    of the Board objects with that error in the previous commit
+                lines: List of errors/warning for this commit, each a str
+                line_boards: Dict keyed by error line, containing a list
+                    of the Board objects with that error in this commit
+                char: Character representing error ('') or warning ('w'). The
+                    broken ('+') or fixed ('-') characters are added in this
+                    function
+
+            Returns:
+                Tuple
+                    List of ErrLine objects for 'better' lines
+                    List of ErrLine objects for 'worse' lines
+            """
             better_lines = []
             worse_lines = []
             for line in lines:
                 if line not in base_lines:
-                    worse_lines.append(char + '+' +
-                            _BoardList(line, line_boards) + line)
+                    errline = ErrLine(char + '+', _BoardList(line, line_boards),
+                                      line)
+                    worse_lines.append(errline)
             for line in base_lines:
                 if line not in lines:
-                    better_lines.append(char + '-' +
-                            _BoardList(line, base_line_boards) + line)
+                    errline = ErrLine(char + '-',
+                                      _BoardList(line, base_line_boards), line)
+                    better_lines.append(errline)
             return better_lines, worse_lines
 
         def _CalcConfig(delta, name, config):
@@ -1200,6 +1306,34 @@ class Builder:
                     col = self.col.YELLOW
                 Print('   ' + line, newline=True, colour=col)
 
+        def _OutputErrLines(err_lines, colour):
+            """Output the line of error/warning lines, if not empty
+
+            Also increments self._error_lines if err_lines not empty
+
+            Args:
+                err_lines: List of ErrLine objects, each an error or warning
+                    line, possibly including a list of boards with that
+                    error/warning
+                colour: Colour to use for output
+            """
+            if err_lines:
+                out_list = []
+                for line in err_lines:
+                    boards = ''
+                    names = [board.target for board in line.boards]
+                    board_str = ' '.join(names) if names else ''
+                    if board_str:
+                        out = self.col.Color(colour, line.char + '(')
+                        out += self.col.Color(self.col.MAGENTA, board_str,
+                                              bright=False)
+                        out += self.col.Color(colour, ') %s' % line.errline)
+                    else:
+                        out = self.col.Color(colour, line.char + line.errline)
+                    out_list.append(out)
+                Print('\n'.join(out_list))
+                self._error_lines += 1
+
 
         ok_boards = []      # List of boards fixed since last commit
         warn_boards = []    # List of boards with warnings since last commit
@@ -1230,7 +1364,7 @@ class Builder:
             else:
                 new_boards.append(target)
 
-        # Get a list of errors that have appeared, and disappeared
+        # Get a list of errors and warnings that have appeared, and disappeared
         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
                 self._base_err_line_boards, err_lines, err_line_boards, '')
         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
@@ -1253,18 +1387,10 @@ class Builder:
             for arch, target_list in arch_list.items():
                 Print('%10s: %s' % (arch, target_list))
                 self._error_lines += 1
-            if better_err:
-                Print('\n'.join(better_err), colour=self.col.GREEN)
-                self._error_lines += 1
-            if worse_err:
-                Print('\n'.join(worse_err), colour=self.col.RED)
-                self._error_lines += 1
-            if better_warn:
-                Print('\n'.join(better_warn), colour=self.col.CYAN)
-                self._error_lines += 1
-            if worse_warn:
-                Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
-                self._error_lines += 1
+            _OutputErrLines(better_err, colour=self.col.GREEN)
+            _OutputErrLines(worse_err, colour=self.col.RED)
+            _OutputErrLines(better_warn, colour=self.col.CYAN)
+            _OutputErrLines(worse_warn, colour=self.col.YELLOW)
 
         if show_sizes:
             self.PrintSizeSummary(board_selected, board_dict, show_detail,
@@ -1472,55 +1598,100 @@ class Builder:
         """Get the directory path to the working dir for a thread.
 
         Args:
-            thread_num: Number of thread to check.
+            thread_num: Number of thread to check (-1 for main process, which
+                is treated as 0)
         """
-        return os.path.join(self._working_dir, '%02d' % thread_num)
+        if self.work_in_output:
+            return self._working_dir
+        return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
 
     def _PrepareThread(self, thread_num, setup_git):
         """Prepare the working directory for a thread.
 
         This clones or fetches the repo into the thread's work directory.
+        Optionally, it can create a linked working tree of the repo in the
+        thread's work directory instead.
 
         Args:
             thread_num: Thread number (0, 1, ...)
-            setup_git: True to set up a git repo clone
+            setup_git:
+               'clone' to set up a git clone
+               'worktree' to set up a git worktree
         """
         thread_dir = self.GetThreadDir(thread_num)
         builderthread.Mkdir(thread_dir)
         git_dir = os.path.join(thread_dir, '.git')
 
-        # Clone the repo if it doesn't already exist
-        # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
-        # we have a private index but uses the origin repo's contents?
+        # Create a worktree or a git repo clone for this thread if it
+        # doesn't already exist
         if setup_git and self.git_dir:
             src_dir = os.path.abspath(self.git_dir)
-            if os.path.exists(git_dir):
+            if os.path.isdir(git_dir):
+                # This is a clone of the src_dir repo, we can keep using
+                # it but need to fetch from src_dir.
+                Print('\rFetching repo for thread %d' % thread_num,
+                      newline=False)
                 gitutil.Fetch(git_dir, thread_dir)
-            else:
+                terminal.PrintClear()
+            elif os.path.isfile(git_dir):
+                # This is a worktree of the src_dir repo, we don't need to
+                # create it again or update it in any way.
+                pass
+            elif os.path.exists(git_dir):
+                # Don't know what could trigger this, but we probably
+                # can't create a git worktree/clone here.
+                raise ValueError('Git dir %s exists, but is not a file '
+                                 'or a directory.' % git_dir)
+            elif setup_git == 'worktree':
+                Print('\rChecking out worktree for thread %d' % thread_num,
+                      newline=False)
+                gitutil.AddWorktree(src_dir, thread_dir)
+                terminal.PrintClear()
+            elif setup_git == 'clone' or setup_git == True:
                 Print('\rCloning repo for thread %d' % thread_num,
                       newline=False)
                 gitutil.Clone(src_dir, thread_dir)
-                Print('\r%s\r' % (' ' * 30), newline=False)
+                terminal.PrintClear()
+            else:
+                raise ValueError("Can't setup git repo with %s." % setup_git)
 
     def _PrepareWorkingSpace(self, max_threads, setup_git):
         """Prepare the working directory for use.
 
-        Set up the git repo for each thread.
+        Set up the git repo for each thread. Creates a linked working tree
+        if git-worktree is available, or clones the repo if it isn't.
 
         Args:
-            max_threads: Maximum number of threads we expect to need.
-            setup_git: True to set up a git repo clone
+            max_threads: Maximum number of threads we expect to need. If 0 then
+                1 is set up, since the main process still needs somewhere to
+                work
+            setup_git: True to set up a git worktree or a git clone
         """
         builderthread.Mkdir(self._working_dir)
-        for thread in range(max_threads):
+        if setup_git and self.git_dir:
+            src_dir = os.path.abspath(self.git_dir)
+            if gitutil.CheckWorktreeIsAvailable(src_dir):
+                setup_git = 'worktree'
+                # If we previously added a worktree but the directory for it
+                # got deleted, we need to prune its files from the repo so
+                # that we can check out another in its place.
+                gitutil.PruneWorktrees(src_dir)
+            else:
+                setup_git = 'clone'
+
+        # Always do at least one thread
+        for thread in range(max(max_threads, 1)):
             self._PrepareThread(thread, setup_git)
 
-    def _PrepareOutputSpace(self):
+    def _GetOutputSpaceRemovals(self):
         """Get the output directories ready to receive files.
 
-        We delete any output directories which look like ones we need to
-        create. Having left over directories is confusing when the user wants
-        to check the output manually.
+        Figure out what needs to be deleted in the output directory before it
+        can be used. We only delete old buildman directories which have the
+        expected name pattern. See _GetOutputDir().
+
+        Returns:
+            List of full paths of directories to remove
         """
         if not self.commits:
             return
@@ -1531,12 +1702,26 @@ class Builder:
         to_remove = []
         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
             if dirname not in dir_list:
-                to_remove.append(dirname)
+                leaf = dirname[len(self.base_dir) + 1:]
+                m =  re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
+                if m:
+                    to_remove.append(dirname)
+        return to_remove
+
+    def _PrepareOutputSpace(self):
+        """Get the output directories ready to receive files.
+
+        We delete any output directories which look like ones we need to
+        create. Having left over directories is confusing when the user wants
+        to check the output manually.
+        """
+        to_remove = self._GetOutputSpaceRemovals()
         if to_remove:
-            Print('Removing %d old build directories' % len(to_remove),
+            Print('Removing %d old build directories...' % len(to_remove),
                   newline=False)
             for dirname in to_remove:
                 shutil.rmtree(dirname)
+            terminal.PrintClear()
 
     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
         """Build all commits for a list of boards
@@ -1551,6 +1736,7 @@ class Builder:
             Tuple containing:
                 - number of boards that failed to build
                 - number of boards that issued warnings
+                - list of thread exceptions raised
         """
         self.commit_count = len(commits) if commits else 1
         self.commits = commits
@@ -1564,24 +1750,48 @@ class Builder:
         Print('\rStarting build...', newline=False)
         self.SetupBuild(board_selected, commits)
         self.ProcessResult(None)
-
+        self.thread_exceptions = []
         # Create jobs to build all commits for each board
         for brd in board_selected.values():
             job = builderthread.BuilderJob()
             job.board = brd
             job.commits = commits
             job.keep_outputs = keep_outputs
+            job.work_in_output = self.work_in_output
+            job.adjust_cfg = self.adjust_cfg
             job.step = self._step
-            self.queue.put(job)
+            if self.num_threads:
+                self.queue.put(job)
+            else:
+                results = self._single_builder.RunJob(job)
 
-        term = threading.Thread(target=self.queue.join)
-        term.setDaemon(True)
-        term.start()
-        while term.isAlive():
-            term.join(100)
+        if self.num_threads:
+            term = threading.Thread(target=self.queue.join)
+            term.setDaemon(True)
+            term.start()
+            while term.is_alive():
+                term.join(100)
 
-        # Wait until we have processed all output
-        self.out_queue.join()
+            # Wait until we have processed all output
+            self.out_queue.join()
         Print()
-        self.ClearLine(0)
-        return (self.fail, self.warned)
+
+        msg = 'Completed: %d total built' % self.count
+        if self.already_done:
+           msg += ' (%d previously' % self.already_done
+           if self.already_done != self.count:
+               msg += ', %d newly' % (self.count - self.already_done)
+           msg += ')'
+        duration = datetime.now() - self._start_time
+        if duration > timedelta(microseconds=1000000):
+            if duration.microseconds >= 500000:
+                duration = duration + timedelta(seconds=1)
+            duration = duration - timedelta(microseconds=duration.microseconds)
+            rate = float(self.count) / duration.total_seconds()
+            msg += ', duration %s, rate %1.2f' % (duration, rate)
+        Print(msg)
+        if self.thread_exceptions:
+            Print('Failed: %d thread exceptions' % len(self.thread_exceptions),
+                  colour=self.col.RED)
+
+        return (self.fail, self.warned, self.thread_exceptions)
This page took 0.049996 seconds and 4 git commands to generate.