tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <[email protected]>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 from buildman import builderthread
  21 from buildman import toolchain
  22 from patman import command
  23 from patman import gitutil
  24 from patman import terminal
  25 from patman.terminal import Print
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185         thread_exceptions: List of exceptions raised by thread jobs
 186
 187     Private members:
 188         _base_board_dict: Last-summarised Dict of boards
 189         _base_err_lines: Last-summarised list of errors
 190         _base_warn_lines: Last-summarised list of warnings
 191         _build_period_us: Time taken for a single build (float object).
 192         _complete_delay: Expected delay until completion (timedelta)
 193         _next_delay_update: Next time we plan to display a progress update
 194                 (datatime)
 195         _show_unknown: Show unknown boards (those not built) in summary
 196         _start_time: Start time for the build
 197         _timestamps: List of timestamps for the completion of the last
 198             last _timestamp_count builds. Each is a datetime object.
 199         _timestamp_count: Number of timestamps to keep in our list.
 200         _working_dir: Base working directory containing all threads
 201         _single_builder: BuilderThread object for the singer builder, if
 202             threading is not being used
 203     """
 204     class Outcome:
 205         """Records a build outcome for a single make invocation
 206
 207         Public Members:
 208             rc: Outcome value (OUTCOME_...)
 209             err_lines: List of error lines or [] if none
 210             sizes: Dictionary of image size information, keyed by filename
 211                 - Each value is itself a dictionary containing
 212                     values for 'text', 'data' and 'bss', being the integer
 213                     size in bytes of each section.
 214             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 215                     value is itself a dictionary:
 216                         key: function name
 217                         value: Size of function in bytes
 218             config: Dictionary keyed by filename - e.g. '.config'. Each
 219                     value is itself a dictionary:
 220                         key: config name
 221                         value: config value
 222             environment: Dictionary keyed by environment variable, Each
 223                      value is the value of environment variable.
 224         """
 225         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 226                      environment):
 227             self.rc = rc
 228             self.err_lines = err_lines
 229             self.sizes = sizes
 230             self.func_sizes = func_sizes
 231             self.config = config
 232             self.environment = environment
 233
 234     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 235                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 236                  no_subdirs=False, full_path=False, verbose_build=False,
 237                  mrproper=False, per_board_out_dir=False,
 238                  config_only=False, squash_config_y=False,
 239                  warnings_as_errors=False, work_in_output=False,
 240                  test_thread_exceptions=False):
 241         """Create a new Builder object
 242
 243         Args:
 244             toolchains: Toolchains object to use for building
 245             base_dir: Base directory to use for builder
 246             git_dir: Git directory containing source repository
 247             num_threads: Number of builder threads to run
 248             num_jobs: Number of jobs to run at once (passed to make as -j)
 249             gnu_make: the command name of GNU Make.
 250             checkout: True to check out source, False to skip that step.
 251                 This is used for testing.
 252             show_unknown: Show unknown boards (those not built) in summary
 253             step: 1 to process every commit, n to process every nth commit
 254             no_subdirs: Don't create subdirectories when building current
 255                 source for a single board
 256             full_path: Return the full path in CROSS_COMPILE and don't set
 257                 PATH
 258             verbose_build: Run build with V=1 and don't use 'make -s'
 259             mrproper: Always run 'make mrproper' when configuring
 260             per_board_out_dir: Build in a separate persistent directory per
 261                 board rather than a thread-specific directory
 262             config_only: Only configure each build, don't build it
 263             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 264             warnings_as_errors: Treat all compiler warnings as errors
 265             work_in_output: Use the output directory as the work directory and
 266                 don't write to a separate output directory.
 267             test_thread_exceptions: Uses for tests only, True to make the
 268                 threads raise an exception instead of reporting their result.
 269                 This simulates a failure in the code somewhere
 270         """
 271         self.toolchains = toolchains
 272         self.base_dir = base_dir
 273         if work_in_output:
 274             self._working_dir = base_dir
 275         else:
 276             self._working_dir = os.path.join(base_dir, '.bm-work')
 277         self.threads = []
 278         self.do_make = self.Make
 279         self.gnu_make = gnu_make
 280         self.checkout = checkout
 281         self.num_threads = num_threads
 282         self.num_jobs = num_jobs
 283         self.already_done = 0
 284         self.force_build = False
 285         self.git_dir = git_dir
 286         self._show_unknown = show_unknown
 287         self._timestamp_count = 10
 288         self._build_period_us = None
 289         self._complete_delay = None
 290         self._next_delay_update = datetime.now()
 291         self._start_time = datetime.now()
 292         self.force_config_on_failure = True
 293         self.force_build_failures = False
 294         self.force_reconfig = False
 295         self._step = step
 296         self.in_tree = False
 297         self._error_lines = 0
 298         self.no_subdirs = no_subdirs
 299         self.full_path = full_path
 300         self.verbose_build = verbose_build
 301         self.config_only = config_only
 302         self.squash_config_y = squash_config_y
 303         self.config_filenames = BASE_CONFIG_FILENAMES
 304         self.work_in_output = work_in_output
 305         if not self.squash_config_y:
 306             self.config_filenames += EXTRA_CONFIG_FILENAMES
 307
 308         self.warnings_as_errors = warnings_as_errors
 309         self.col = terminal.Color()
 310
 311         self._re_function = re.compile('(.*): In function.*')
 312         self._re_files = re.compile('In file included from.*')
 313         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 314         self._re_dtb_warning = re.compile('(.*): Warning .*')
 315         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 316         self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
 317                                                 re.MULTILINE | re.DOTALL)
 318
 319         self.thread_exceptions = []
 320         self.test_thread_exceptions = test_thread_exceptions
 321         if self.num_threads:
 322             self._single_builder = None
 323             self.queue = queue.Queue()
 324             self.out_queue = queue.Queue()
 325             for i in range(self.num_threads):
 326                 t = builderthread.BuilderThread(
 327                         self, i, mrproper, per_board_out_dir,
 328                         test_exception=test_thread_exceptions)
 329                 t.setDaemon(True)
 330                 t.start()
 331                 self.threads.append(t)
 332
 333             t = builderthread.ResultThread(self)
 334             t.setDaemon(True)
 335             t.start()
 336             self.threads.append(t)
 337         else:
 338             self._single_builder = builderthread.BuilderThread(
 339                 self, -1, mrproper, per_board_out_dir)
 340
 341         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 342         self.re_make_err = re.compile('|'.join(ignore_lines))
 343
 344         # Handle existing graceful with SIGINT / Ctrl-C
 345         signal.signal(signal.SIGINT, self.signal_handler)
 346
 347     def __del__(self):
 348         """Get rid of all threads created by the builder"""
 349         for t in self.threads:
 350             del t
 351
 352     def signal_handler(self, signal, frame):
 353         sys.exit(1)
 354
 355     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 356                           show_detail=False, show_bloat=False,
 357                           list_error_boards=False, show_config=False,
 358                           show_environment=False, filter_dtb_warnings=False,
 359                           filter_migration_warnings=False):
 360         """Setup display options for the builder.
 361
 362         Args:
 363             show_errors: True to show summarised error/warning info
 364             show_sizes: Show size deltas
 365             show_detail: Show size delta detail for each board if show_sizes
 366             show_bloat: Show detail for each function
 367             list_error_boards: Show the boards which caused each error/warning
 368             show_config: Show config deltas
 369             show_environment: Show environment deltas
 370             filter_dtb_warnings: Filter out any warnings from the device-tree
 371                 compiler
 372             filter_migration_warnings: Filter out any warnings about migrating
 373                 a board to driver model
 374         """
 375         self._show_errors = show_errors
 376         self._show_sizes = show_sizes
 377         self._show_detail = show_detail
 378         self._show_bloat = show_bloat
 379         self._list_error_boards = list_error_boards
 380         self._show_config = show_config
 381         self._show_environment = show_environment
 382         self._filter_dtb_warnings = filter_dtb_warnings
 383         self._filter_migration_warnings = filter_migration_warnings
 384
 385     def _AddTimestamp(self):
 386         """Add a new timestamp to the list and record the build period.
 387
 388         The build period is the length of time taken to perform a single
 389         build (one board, one commit).
 390         """
 391         now = datetime.now()
 392         self._timestamps.append(now)
 393         count = len(self._timestamps)
 394         delta = self._timestamps[-1] - self._timestamps[0]
 395         seconds = delta.total_seconds()
 396
 397         # If we have enough data, estimate build period (time taken for a
 398         # single build) and therefore completion time.
 399         if count > 1 and self._next_delay_update < now:
 400             self._next_delay_update = now + timedelta(seconds=2)
 401             if seconds > 0:
 402                 self._build_period = float(seconds) / count
 403                 todo = self.count - self.upto
 404                 self._complete_delay = timedelta(microseconds=
 405                         self._build_period * todo * 1000000)
 406                 # Round it
 407                 self._complete_delay -= timedelta(
 408                         microseconds=self._complete_delay.microseconds)
 409
 410         if seconds > 60:
 411             self._timestamps.popleft()
 412             count -= 1
 413
 414     def SelectCommit(self, commit, checkout=True):
 415         """Checkout the selected commit for this build
 416         """
 417         self.commit = commit
 418         if checkout and self.checkout:
 419             gitutil.Checkout(commit.hash)
 420
 421     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 422         """Run make
 423
 424         Args:
 425             commit: Commit object that is being built
 426             brd: Board object that is being built
 427             stage: Stage that we are at (mrproper, config, build)
 428             cwd: Directory where make should be run
 429             args: Arguments to pass to make
 430             kwargs: Arguments to pass to command.RunPipe()
 431         """
 432         cmd = [self.gnu_make] + list(args)
 433         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 434                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 435         if self.verbose_build:
 436             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 437             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 438         return result
 439
 440     def ProcessResult(self, result):
 441         """Process the result of a build, showing progress information
 442
 443         Args:
 444             result: A CommandResult object, which indicates the result for
 445                     a single build
 446         """
 447         col = terminal.Color()
 448         if result:
 449             target = result.brd.target
 450
 451             self.upto += 1
 452             if result.return_code != 0:
 453                 self.fail += 1
 454             elif result.stderr:
 455                 self.warned += 1
 456             if result.already_done:
 457                 self.already_done += 1
 458             if self._verbose:
 459                 terminal.PrintClear()
 460                 boards_selected = {target : result.brd}
 461                 self.ResetResultSummary(boards_selected)
 462                 self.ProduceResultSummary(result.commit_upto, self.commits,
 463                                           boards_selected)
 464         else:
 465             target = '(starting)'
 466
 467         # Display separate counts for ok, warned and fail
 468         ok = self.upto - self.warned - self.fail
 469         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 470         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 471         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 472
 473         line += ' /%-5d  ' % self.count
 474         remaining = self.count - self.upto
 475         if remaining:
 476             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 477         else:
 478             line += ' ' * 8
 479
 480         # Add our current completion time estimate
 481         self._AddTimestamp()
 482         if self._complete_delay:
 483             line += '%s  : ' % self._complete_delay
 484
 485         line += target
 486         terminal.PrintClear()
 487         Print(line, newline=False, limit_to_line=True)
 488
 489     def _GetOutputDir(self, commit_upto):
 490         """Get the name of the output directory for a commit number
 491
 492         The output directory is typically .../<branch>/<commit>.
 493
 494         Args:
 495             commit_upto: Commit number to use (0..self.count-1)
 496         """
 497         if self.work_in_output:
 498             return self._working_dir
 499
 500         commit_dir = None
 501         if self.commits:
 502             commit = self.commits[commit_upto]
 503             subject = commit.subject.translate(trans_valid_chars)
 504             # See _GetOutputSpaceRemovals() which parses this name
 505             commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
 506                     commit.hash, subject[:20]))
 507         elif not self.no_subdirs:
 508             commit_dir = 'current'
 509         if not commit_dir:
 510             return self.base_dir
 511         return os.path.join(self.base_dir, commit_dir)
 512
 513     def GetBuildDir(self, commit_upto, target):
 514         """Get the name of the build directory for a commit number
 515
 516         The build directory is typically .../<branch>/<commit>/<target>.
 517
 518         Args:
 519             commit_upto: Commit number to use (0..self.count-1)
 520             target: Target name
 521         """
 522         output_dir = self._GetOutputDir(commit_upto)
 523         if self.work_in_output:
 524             return output_dir
 525         return os.path.join(output_dir, target)
 526
 527     def GetDoneFile(self, commit_upto, target):
 528         """Get the name of the done file for a commit number
 529
 530         Args:
 531             commit_upto: Commit number to use (0..self.count-1)
 532             target: Target name
 533         """
 534         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 535
 536     def GetSizesFile(self, commit_upto, target):
 537         """Get the name of the sizes file for a commit number
 538
 539         Args:
 540             commit_upto: Commit number to use (0..self.count-1)
 541             target: Target name
 542         """
 543         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 544
 545     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 546         """Get the name of the funcsizes file for a commit number and ELF file
 547
 548         Args:
 549             commit_upto: Commit number to use (0..self.count-1)
 550             target: Target name
 551             elf_fname: Filename of elf image
 552         """
 553         return os.path.join(self.GetBuildDir(commit_upto, target),
 554                             '%s.sizes' % elf_fname.replace('/', '-'))
 555
 556     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 557         """Get the name of the objdump file for a commit number and ELF file
 558
 559         Args:
 560             commit_upto: Commit number to use (0..self.count-1)
 561             target: Target name
 562             elf_fname: Filename of elf image
 563         """
 564         return os.path.join(self.GetBuildDir(commit_upto, target),
 565                             '%s.objdump' % elf_fname.replace('/', '-'))
 566
 567     def GetErrFile(self, commit_upto, target):
 568         """Get the name of the err file for a commit number
 569
 570         Args:
 571             commit_upto: Commit number to use (0..self.count-1)
 572             target: Target name
 573         """
 574         output_dir = self.GetBuildDir(commit_upto, target)
 575         return os.path.join(output_dir, 'err')
 576
 577     def FilterErrors(self, lines):
 578         """Filter out errors in which we have no interest
 579
 580         We should probably use map().
 581
 582         Args:
 583             lines: List of error lines, each a string
 584         Returns:
 585             New list with only interesting lines included
 586         """
 587         out_lines = []
 588         if self._filter_migration_warnings:
 589             text = '\n'.join(lines)
 590             text = self._re_migration_warning.sub('', text)
 591             lines = text.splitlines()
 592         for line in lines:
 593             if self.re_make_err.search(line):
 594                 continue
 595             if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
 596                 continue
 597             out_lines.append(line)
 598         return out_lines
 599
 600     def ReadFuncSizes(self, fname, fd):
 601         """Read function sizes from the output of 'nm'
 602
 603         Args:
 604             fd: File containing data to read
 605             fname: Filename we are reading from (just for errors)
 606
 607         Returns:
 608             Dictionary containing size of each function in bytes, indexed by
 609             function name.
 610         """
 611         sym = {}
 612         for line in fd.readlines():
 613             try:
 614                 if line.strip():
 615                     size, type, name = line[:-1].split()
 616             except:
 617                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 618                 continue
 619             if type in 'tTdDbB':
 620                 # function names begin with '.' on 64-bit powerpc
 621                 if '.' in name[1:]:
 622                     name = 'static.' + name.split('.')[0]
 623                 sym[name] = sym.get(name, 0) + int(size, 16)
 624         return sym
 625
 626     def _ProcessConfig(self, fname):
 627         """Read in a .config, autoconf.mk or autoconf.h file
 628
 629         This function handles all config file types. It ignores comments and
 630         any #defines which don't start with CONFIG_.
 631
 632         Args:
 633             fname: Filename to read
 634
 635         Returns:
 636             Dictionary:
 637                 key: Config name (e.g. CONFIG_DM)
 638                 value: Config value (e.g. 1)
 639         """
 640         config = {}
 641         if os.path.exists(fname):
 642             with open(fname) as fd:
 643                 for line in fd:
 644                     line = line.strip()
 645                     if line.startswith('#define'):
 646                         values = line[8:].split(' ', 1)
 647                         if len(values) > 1:
 648                             key, value = values
 649                         else:
 650                             key = values[0]
 651                             value = '1' if self.squash_config_y else ''
 652                         if not key.startswith('CONFIG_'):
 653                             continue
 654                     elif not line or line[0] in ['#', '*', '/']:
 655                         continue
 656                     else:
 657                         key, value = line.split('=', 1)
 658                     if self.squash_config_y and value == 'y':
 659                         value = '1'
 660                     config[key] = value
 661         return config
 662
 663     def _ProcessEnvironment(self, fname):
 664         """Read in a uboot.env file
 665
 666         This function reads in environment variables from a file.
 667
 668         Args:
 669             fname: Filename to read
 670
 671         Returns:
 672             Dictionary:
 673                 key: environment variable (e.g. bootlimit)
 674                 value: value of environment variable (e.g. 1)
 675         """
 676         environment = {}
 677         if os.path.exists(fname):
 678             with open(fname) as fd:
 679                 for line in fd.read().split('\0'):
 680                     try:
 681                         key, value = line.split('=', 1)
 682                         environment[key] = value
 683                     except ValueError:
 684                         # ignore lines we can't parse
 685                         pass
 686         return environment
 687
 688     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 689                         read_config, read_environment):
 690         """Work out the outcome of a build.
 691
 692         Args:
 693             commit_upto: Commit number to check (0..n-1)
 694             target: Target board to check
 695             read_func_sizes: True to read function size information
 696             read_config: True to read .config and autoconf.h files
 697             read_environment: True to read uboot.env files
 698
 699         Returns:
 700             Outcome object
 701         """
 702         done_file = self.GetDoneFile(commit_upto, target)
 703         sizes_file = self.GetSizesFile(commit_upto, target)
 704         sizes = {}
 705         func_sizes = {}
 706         config = {}
 707         environment = {}
 708         if os.path.exists(done_file):
 709             with open(done_file, 'r') as fd:
 710                 try:
 711                     return_code = int(fd.readline())
 712                 except ValueError:
 713                     # The file may be empty due to running out of disk space.
 714                     # Try a rebuild
 715                     return_code = 1
 716                 err_lines = []
 717                 err_file = self.GetErrFile(commit_upto, target)
 718                 if os.path.exists(err_file):
 719                     with open(err_file, 'r') as fd:
 720                         err_lines = self.FilterErrors(fd.readlines())
 721
 722                 # Decide whether the build was ok, failed or created warnings
 723                 if return_code:
 724                     rc = OUTCOME_ERROR
 725                 elif len(err_lines):
 726                     rc = OUTCOME_WARNING
 727                 else:
 728                     rc = OUTCOME_OK
 729
 730                 # Convert size information to our simple format
 731                 if os.path.exists(sizes_file):
 732                     with open(sizes_file, 'r') as fd:
 733                         for line in fd.readlines():
 734                             values = line.split()
 735                             rodata = 0
 736                             if len(values) > 6:
 737                                 rodata = int(values[6], 16)
 738                             size_dict = {
 739                                 'all' : int(values[0]) + int(values[1]) +
 740                                         int(values[2]),
 741                                 'text' : int(values[0]) - rodata,
 742                                 'data' : int(values[1]),
 743                                 'bss' : int(values[2]),
 744                                 'rodata' : rodata,
 745                             }
 746                             sizes[values[5]] = size_dict
 747
 748             if read_func_sizes:
 749                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 750                 for fname in glob.glob(pattern):
 751                     with open(fname, 'r') as fd:
 752                         dict_name = os.path.basename(fname).replace('.sizes',
 753                                                                     '')
 754                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 755
 756             if read_config:
 757                 output_dir = self.GetBuildDir(commit_upto, target)
 758                 for name in self.config_filenames:
 759                     fname = os.path.join(output_dir, name)
 760                     config[name] = self._ProcessConfig(fname)
 761
 762             if read_environment:
 763                 output_dir = self.GetBuildDir(commit_upto, target)
 764                 fname = os.path.join(output_dir, 'uboot.env')
 765                 environment = self._ProcessEnvironment(fname)
 766
 767             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 768                                    environment)
 769
 770         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 771
 772     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 773                          read_config, read_environment):
 774         """Calculate a summary of the results of building a commit.
 775
 776         Args:
 777             board_selected: Dict containing boards to summarise
 778             commit_upto: Commit number to summarize (0..self.count-1)
 779             read_func_sizes: True to read function size information
 780             read_config: True to read .config and autoconf.h files
 781             read_environment: True to read uboot.env files
 782
 783         Returns:
 784             Tuple:
 785                 Dict containing boards which passed building this commit.
 786                     keyed by board.target
 787                 List containing a summary of error lines
 788                 Dict keyed by error line, containing a list of the Board
 789                     objects with that error
 790                 List containing a summary of warning lines
 791                 Dict keyed by error line, containing a list of the Board
 792                     objects with that warning
 793                 Dictionary keyed by board.target. Each value is a dictionary:
 794                     key: filename - e.g. '.config'
 795                     value is itself a dictionary:
 796                         key: config name
 797                         value: config value
 798                 Dictionary keyed by board.target. Each value is a dictionary:
 799                     key: environment variable
 800                     value: value of environment variable
 801         """
 802         def AddLine(lines_summary, lines_boards, line, board):
 803             line = line.rstrip()
 804             if line in lines_boards:
 805                 lines_boards[line].append(board)
 806             else:
 807                 lines_boards[line] = [board]
 808                 lines_summary.append(line)
 809
 810         board_dict = {}
 811         err_lines_summary = []
 812         err_lines_boards = {}
 813         warn_lines_summary = []
 814         warn_lines_boards = {}
 815         config = {}
 816         environment = {}
 817
 818         for board in boards_selected.values():
 819             outcome = self.GetBuildOutcome(commit_upto, board.target,
 820                                            read_func_sizes, read_config,
 821                                            read_environment)
 822             board_dict[board.target] = outcome
 823             last_func = None
 824             last_was_warning = False
 825             for line in outcome.err_lines:
 826                 if line:
 827                     if (self._re_function.match(line) or
 828                             self._re_files.match(line)):
 829                         last_func = line
 830                     else:
 831                         is_warning = (self._re_warning.match(line) or
 832                                       self._re_dtb_warning.match(line))
 833                         is_note = self._re_note.match(line)
 834                         if is_warning or (last_was_warning and is_note):
 835                             if last_func:
 836                                 AddLine(warn_lines_summary, warn_lines_boards,
 837                                         last_func, board)
 838                             AddLine(warn_lines_summary, warn_lines_boards,
 839                                     line, board)
 840                         else:
 841                             if last_func:
 842                                 AddLine(err_lines_summary, err_lines_boards,
 843                                         last_func, board)
 844                             AddLine(err_lines_summary, err_lines_boards,
 845                                     line, board)
 846                         last_was_warning = is_warning
 847                         last_func = None
 848             tconfig = Config(self.config_filenames, board.target)
 849             for fname in self.config_filenames:
 850                 if outcome.config:
 851                     for key, value in outcome.config[fname].items():
 852                         tconfig.Add(fname, key, value)
 853             config[board.target] = tconfig
 854
 855             tenvironment = Environment(board.target)
 856             if outcome.environment:
 857                 for key, value in outcome.environment.items():
 858                     tenvironment.Add(key, value)
 859             environment[board.target] = tenvironment
 860
 861         return (board_dict, err_lines_summary, err_lines_boards,
 862                 warn_lines_summary, warn_lines_boards, config, environment)
 863
 864     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 865         """Add an output to our list of outcomes for each architecture
 866
 867         This simple function adds failing boards (changes) to the
 868         relevant architecture string, so we can print the results out
 869         sorted by architecture.
 870
 871         Args:
 872              board_dict: Dict containing all boards
 873              arch_list: Dict keyed by arch name. Value is a string containing
 874                     a list of board names which failed for that arch.
 875              changes: List of boards to add to arch_list
 876              color: terminal.Colour object
 877         """
 878         done_arch = {}
 879         for target in changes:
 880             if target in board_dict:
 881                 arch = board_dict[target].arch
 882             else:
 883                 arch = 'unknown'
 884             str = self.col.Color(color, ' ' + target)
 885             if not arch in done_arch:
 886                 str = ' %s  %s' % (self.col.Color(color, char), str)
 887                 done_arch[arch] = True
 888             if not arch in arch_list:
 889                 arch_list[arch] = str
 890             else:
 891                 arch_list[arch] += str
 892
 893
 894     def ColourNum(self, num):
 895         color = self.col.RED if num > 0 else self.col.GREEN
 896         if num == 0:
 897             return '0'
 898         return self.col.Color(color, str(num))
 899
 900     def ResetResultSummary(self, board_selected):
 901         """Reset the results summary ready for use.
 902
 903         Set up the base board list to be all those selected, and set the
 904         error lines to empty.
 905
 906         Following this, calls to PrintResultSummary() will use this
 907         information to work out what has changed.
 908
 909         Args:
 910             board_selected: Dict containing boards to summarise, keyed by
 911                 board.target
 912         """
 913         self._base_board_dict = {}
 914         for board in board_selected:
 915             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 916                                                            {})
 917         self._base_err_lines = []
 918         self._base_warn_lines = []
 919         self._base_err_line_boards = {}
 920         self._base_warn_line_boards = {}
 921         self._base_config = None
 922         self._base_environment = None
 923
 924     def PrintFuncSizeDetail(self, fname, old, new):
 925         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 926         delta, common = [], {}
 927
 928         for a in old:
 929             if a in new:
 930                 common[a] = 1
 931
 932         for name in old:
 933             if name not in common:
 934                 remove += 1
 935                 down += old[name]
 936                 delta.append([-old[name], name])
 937
 938         for name in new:
 939             if name not in common:
 940                 add += 1
 941                 up += new[name]
 942                 delta.append([new[name], name])
 943
 944         for name in common:
 945                 diff = new.get(name, 0) - old.get(name, 0)
 946                 if diff > 0:
 947                     grow, up = grow + 1, up + diff
 948                 elif diff < 0:
 949                     shrink, down = shrink + 1, down - diff
 950                 delta.append([diff, name])
 951
 952         delta.sort()
 953         delta.reverse()
 954
 955         args = [add, -remove, grow, -shrink, up, -down, up - down]
 956         if max(args) == 0 and min(args) == 0:
 957             return
 958         args = [self.ColourNum(x) for x in args]
 959         indent = ' ' * 15
 960         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 961               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 962         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 963                                          'delta'))
 964         for diff, name in delta:
 965             if diff:
 966                 color = self.col.RED if diff > 0 else self.col.GREEN
 967                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 968                         old.get(name, '-'), new.get(name,'-'), diff)
 969                 Print(msg, colour=color)
 970
 971
 972     def PrintSizeDetail(self, target_list, show_bloat):
 973         """Show details size information for each board
 974
 975         Args:
 976             target_list: List of targets, each a dict containing:
 977                     'target': Target name
 978                     'total_diff': Total difference in bytes across all areas
 979                     <part_name>: Difference for that part
 980             show_bloat: Show detail for each function
 981         """
 982         targets_by_diff = sorted(target_list, reverse=True,
 983         key=lambda x: x['_total_diff'])
 984         for result in targets_by_diff:
 985             printed_target = False
 986             for name in sorted(result):
 987                 diff = result[name]
 988                 if name.startswith('_'):
 989                     continue
 990                 if diff != 0:
 991                     color = self.col.RED if diff > 0 else self.col.GREEN
 992                 msg = ' %s %+d' % (name, diff)
 993                 if not printed_target:
 994                     Print('%10s  %-15s:' % ('', result['_target']),
 995                           newline=False)
 996                     printed_target = True
 997                 Print(msg, colour=color, newline=False)
 998             if printed_target:
 999                 Print()
1000                 if show_bloat:
1001                     target = result['_target']
1002                     outcome = result['_outcome']
1003                     base_outcome = self._base_board_dict[target]
1004                     for fname in outcome.func_sizes:
1005                         self.PrintFuncSizeDetail(fname,
1006                                                  base_outcome.func_sizes[fname],
1007                                                  outcome.func_sizes[fname])
1008
1009
1010     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
1011                          show_bloat):
1012         """Print a summary of image sizes broken down by section.
1013
1014         The summary takes the form of one line per architecture. The
1015         line contains deltas for each of the sections (+ means the section
1016         got bigger, - means smaller). The numbers are the average number
1017         of bytes that a board in this section increased by.
1018
1019         For example:
1020            powerpc: (622 boards)   text -0.0
1021           arm: (285 boards)   text -0.0
1022           nds32: (3 boards)   text -8.0
1023
1024         Args:
1025             board_selected: Dict containing boards to summarise, keyed by
1026                 board.target
1027             board_dict: Dict containing boards for which we built this
1028                 commit, keyed by board.target. The value is an Outcome object.
1029             show_detail: Show size delta detail for each board
1030             show_bloat: Show detail for each function
1031         """
1032         arch_list = {}
1033         arch_count = {}
1034
1035         # Calculate changes in size for different image parts
1036         # The previous sizes are in Board.sizes, for each board
1037         for target in board_dict:
1038             if target not in board_selected:
1039                 continue
1040             base_sizes = self._base_board_dict[target].sizes
1041             outcome = board_dict[target]
1042             sizes = outcome.sizes
1043
1044             # Loop through the list of images, creating a dict of size
1045             # changes for each image/part. We end up with something like
1046             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1047             # which means that U-Boot data increased by 5 bytes and SPL
1048             # text decreased by 4.
1049             err = {'_target' : target}
1050             for image in sizes:
1051                 if image in base_sizes:
1052                     base_image = base_sizes[image]
1053                     # Loop through the text, data, bss parts
1054                     for part in sorted(sizes[image]):
1055                         diff = sizes[image][part] - base_image[part]
1056                         col = None
1057                         if diff:
1058                             if image == 'u-boot':
1059                                 name = part
1060                             else:
1061                                 name = image + ':' + part
1062                             err[name] = diff
1063             arch = board_selected[target].arch
1064             if not arch in arch_count:
1065                 arch_count[arch] = 1
1066             else:
1067                 arch_count[arch] += 1
1068             if not sizes:
1069                 pass    # Only add to our list when we have some stats
1070             elif not arch in arch_list:
1071                 arch_list[arch] = [err]
1072             else:
1073                 arch_list[arch].append(err)
1074
1075         # We now have a list of image size changes sorted by arch
1076         # Print out a summary of these
1077         for arch, target_list in arch_list.items():
1078             # Get total difference for each type
1079             totals = {}
1080             for result in target_list:
1081                 total = 0
1082                 for name, diff in result.items():
1083                     if name.startswith('_'):
1084                         continue
1085                     total += diff
1086                     if name in totals:
1087                         totals[name] += diff
1088                     else:
1089                         totals[name] = diff
1090                 result['_total_diff'] = total
1091                 result['_outcome'] = board_dict[result['_target']]
1092
1093             count = len(target_list)
1094             printed_arch = False
1095             for name in sorted(totals):
1096                 diff = totals[name]
1097                 if diff:
1098                     # Display the average difference in this name for this
1099                     # architecture
1100                     avg_diff = float(diff) / count
1101                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1102                     msg = ' %s %+1.1f' % (name, avg_diff)
1103                     if not printed_arch:
1104                         Print('%10s: (for %d/%d boards)' % (arch, count,
1105                               arch_count[arch]), newline=False)
1106                         printed_arch = True
1107                     Print(msg, colour=color, newline=False)
1108
1109             if printed_arch:
1110                 Print()
1111                 if show_detail:
1112                     self.PrintSizeDetail(target_list, show_bloat)
1113
1114
1115     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1116                            err_line_boards, warn_lines, warn_line_boards,
1117                            config, environment, show_sizes, show_detail,
1118                            show_bloat, show_config, show_environment):
1119         """Compare results with the base results and display delta.
1120
1121         Only boards mentioned in board_selected will be considered. This
1122         function is intended to be called repeatedly with the results of
1123         each commit. It therefore shows a 'diff' between what it saw in
1124         the last call and what it sees now.
1125
1126         Args:
1127             board_selected: Dict containing boards to summarise, keyed by
1128                 board.target
1129             board_dict: Dict containing boards for which we built this
1130                 commit, keyed by board.target. The value is an Outcome object.
1131             err_lines: A list of errors for this commit, or [] if there is
1132                 none, or we don't want to print errors
1133             err_line_boards: Dict keyed by error line, containing a list of
1134                 the Board objects with that error
1135             warn_lines: A list of warnings for this commit, or [] if there is
1136                 none, or we don't want to print errors
1137             warn_line_boards: Dict keyed by warning line, containing a list of
1138                 the Board objects with that warning
1139             config: Dictionary keyed by filename - e.g. '.config'. Each
1140                     value is itself a dictionary:
1141                         key: config name
1142                         value: config value
1143             environment: Dictionary keyed by environment variable, Each
1144                      value is the value of environment variable.
1145             show_sizes: Show image size deltas
1146             show_detail: Show size delta detail for each board if show_sizes
1147             show_bloat: Show detail for each function
1148             show_config: Show config changes
1149             show_environment: Show environment changes
1150         """
1151         def _BoardList(line, line_boards):
1152             """Helper function to get a line of boards containing a line
1153
1154             Args:
1155                 line: Error line to search for
1156                 line_boards: boards to search, each a Board
1157             Return:
1158                 List of boards with that error line, or [] if the user has not
1159                     requested such a list
1160             """
1161             boards = []
1162             board_set = set()
1163             if self._list_error_boards:
1164                 for board in line_boards[line]:
1165                     if not board in board_set:
1166                         boards.append(board)
1167                         board_set.add(board)
1168             return boards
1169
1170         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1171                             char):
1172             """Calculate the required output based on changes in errors
1173
1174             Args:
1175                 base_lines: List of errors/warnings for previous commit
1176                 base_line_boards: Dict keyed by error line, containing a list
1177                     of the Board objects with that error in the previous commit
1178                 lines: List of errors/warning for this commit, each a str
1179                 line_boards: Dict keyed by error line, containing a list
1180                     of the Board objects with that error in this commit
1181                 char: Character representing error ('') or warning ('w'). The
1182                     broken ('+') or fixed ('-') characters are added in this
1183                     function
1184
1185             Returns:
1186                 Tuple
1187                     List of ErrLine objects for 'better' lines
1188                     List of ErrLine objects for 'worse' lines
1189             """
1190             better_lines = []
1191             worse_lines = []
1192             for line in lines:
1193                 if line not in base_lines:
1194                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1195                                       line)
1196                     worse_lines.append(errline)
1197             for line in base_lines:
1198                 if line not in lines:
1199                     errline = ErrLine(char + '-',
1200                                       _BoardList(line, base_line_boards), line)
1201                     better_lines.append(errline)
1202             return better_lines, worse_lines
1203
1204         def _CalcConfig(delta, name, config):
1205             """Calculate configuration changes
1206
1207             Args:
1208                 delta: Type of the delta, e.g. '+'
1209                 name: name of the file which changed (e.g. .config)
1210                 config: configuration change dictionary
1211                     key: config name
1212                     value: config value
1213             Returns:
1214                 String containing the configuration changes which can be
1215                     printed
1216             """
1217             out = ''
1218             for key in sorted(config.keys()):
1219                 out += '%s=%s ' % (key, config[key])
1220             return '%s %s: %s' % (delta, name, out)
1221
1222         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1223             """Add changes in configuration to a list
1224
1225             Args:
1226                 lines: list to add to
1227                 name: config file name
1228                 config_plus: configurations added, dictionary
1229                     key: config name
1230                     value: config value
1231                 config_minus: configurations removed, dictionary
1232                     key: config name
1233                     value: config value
1234                 config_change: configurations changed, dictionary
1235                     key: config name
1236                     value: config value
1237             """
1238             if config_plus:
1239                 lines.append(_CalcConfig('+', name, config_plus))
1240             if config_minus:
1241                 lines.append(_CalcConfig('-', name, config_minus))
1242             if config_change:
1243                 lines.append(_CalcConfig('c', name, config_change))
1244
1245         def _OutputConfigInfo(lines):
1246             for line in lines:
1247                 if not line:
1248                     continue
1249                 if line[0] == '+':
1250                     col = self.col.GREEN
1251                 elif line[0] == '-':
1252                     col = self.col.RED
1253                 elif line[0] == 'c':
1254                     col = self.col.YELLOW
1255                 Print('   ' + line, newline=True, colour=col)
1256
1257         def _OutputErrLines(err_lines, colour):
1258             """Output the line of error/warning lines, if not empty
1259
1260             Also increments self._error_lines if err_lines not empty
1261
1262             Args:
1263                 err_lines: List of ErrLine objects, each an error or warning
1264                     line, possibly including a list of boards with that
1265                     error/warning
1266                 colour: Colour to use for output
1267             """
1268             if err_lines:
1269                 out_list = []
1270                 for line in err_lines:
1271                     boards = ''
1272                     names = [board.target for board in line.boards]
1273                     board_str = ' '.join(names) if names else ''
1274                     if board_str:
1275                         out = self.col.Color(colour, line.char + '(')
1276                         out += self.col.Color(self.col.MAGENTA, board_str,
1277                                               bright=False)
1278                         out += self.col.Color(colour, ') %s' % line.errline)
1279                     else:
1280                         out = self.col.Color(colour, line.char + line.errline)
1281                     out_list.append(out)
1282                 Print('\n'.join(out_list))
1283                 self._error_lines += 1
1284
1285
1286         ok_boards = []      # List of boards fixed since last commit
1287         warn_boards = []    # List of boards with warnings since last commit
1288         err_boards = []     # List of new broken boards since last commit
1289         new_boards = []     # List of boards that didn't exist last time
1290         unknown_boards = [] # List of boards that were not built
1291
1292         for target in board_dict:
1293             if target not in board_selected:
1294                 continue
1295
1296             # If the board was built last time, add its outcome to a list
1297             if target in self._base_board_dict:
1298                 base_outcome = self._base_board_dict[target].rc
1299                 outcome = board_dict[target]
1300                 if outcome.rc == OUTCOME_UNKNOWN:
1301                     unknown_boards.append(target)
1302                 elif outcome.rc < base_outcome:
1303                     if outcome.rc == OUTCOME_WARNING:
1304                         warn_boards.append(target)
1305                     else:
1306                         ok_boards.append(target)
1307                 elif outcome.rc > base_outcome:
1308                     if outcome.rc == OUTCOME_WARNING:
1309                         warn_boards.append(target)
1310                     else:
1311                         err_boards.append(target)
1312             else:
1313                 new_boards.append(target)
1314
1315         # Get a list of errors and warnings that have appeared, and disappeared
1316         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1317                 self._base_err_line_boards, err_lines, err_line_boards, '')
1318         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1319                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1320
1321         # Display results by arch
1322         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1323                 worse_err, better_err, worse_warn, better_warn)):
1324             arch_list = {}
1325             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1326                     self.col.GREEN)
1327             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1328                     self.col.YELLOW)
1329             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1330                     self.col.RED)
1331             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1332             if self._show_unknown:
1333                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1334                         self.col.MAGENTA)
1335             for arch, target_list in arch_list.items():
1336                 Print('%10s: %s' % (arch, target_list))
1337                 self._error_lines += 1
1338             _OutputErrLines(better_err, colour=self.col.GREEN)
1339             _OutputErrLines(worse_err, colour=self.col.RED)
1340             _OutputErrLines(better_warn, colour=self.col.CYAN)
1341             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1342
1343         if show_sizes:
1344             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1345                                   show_bloat)
1346
1347         if show_environment and self._base_environment:
1348             lines = []
1349
1350             for target in board_dict:
1351                 if target not in board_selected:
1352                     continue
1353
1354                 tbase = self._base_environment[target]
1355                 tenvironment = environment[target]
1356                 environment_plus = {}
1357                 environment_minus = {}
1358                 environment_change = {}
1359                 base = tbase.environment
1360                 for key, value in tenvironment.environment.items():
1361                     if key not in base:
1362                         environment_plus[key] = value
1363                 for key, value in base.items():
1364                     if key not in tenvironment.environment:
1365                         environment_minus[key] = value
1366                 for key, value in base.items():
1367                     new_value = tenvironment.environment.get(key)
1368                     if new_value and value != new_value:
1369                         desc = '%s -> %s' % (value, new_value)
1370                         environment_change[key] = desc
1371
1372                 _AddConfig(lines, target, environment_plus, environment_minus,
1373                            environment_change)
1374
1375             _OutputConfigInfo(lines)
1376
1377         if show_config and self._base_config:
1378             summary = {}
1379             arch_config_plus = {}
1380             arch_config_minus = {}
1381             arch_config_change = {}
1382             arch_list = []
1383
1384             for target in board_dict:
1385                 if target not in board_selected:
1386                     continue
1387                 arch = board_selected[target].arch
1388                 if arch not in arch_list:
1389                     arch_list.append(arch)
1390
1391             for arch in arch_list:
1392                 arch_config_plus[arch] = {}
1393                 arch_config_minus[arch] = {}
1394                 arch_config_change[arch] = {}
1395                 for name in self.config_filenames:
1396                     arch_config_plus[arch][name] = {}
1397                     arch_config_minus[arch][name] = {}
1398                     arch_config_change[arch][name] = {}
1399
1400             for target in board_dict:
1401                 if target not in board_selected:
1402                     continue
1403
1404                 arch = board_selected[target].arch
1405
1406                 all_config_plus = {}
1407                 all_config_minus = {}
1408                 all_config_change = {}
1409                 tbase = self._base_config[target]
1410                 tconfig = config[target]
1411                 lines = []
1412                 for name in self.config_filenames:
1413                     if not tconfig.config[name]:
1414                         continue
1415                     config_plus = {}
1416                     config_minus = {}
1417                     config_change = {}
1418                     base = tbase.config[name]
1419                     for key, value in tconfig.config[name].items():
1420                         if key not in base:
1421                             config_plus[key] = value
1422                             all_config_plus[key] = value
1423                     for key, value in base.items():
1424                         if key not in tconfig.config[name]:
1425                             config_minus[key] = value
1426                             all_config_minus[key] = value
1427                     for key, value in base.items():
1428                         new_value = tconfig.config.get(key)
1429                         if new_value and value != new_value:
1430                             desc = '%s -> %s' % (value, new_value)
1431                             config_change[key] = desc
1432                             all_config_change[key] = desc
1433
1434                     arch_config_plus[arch][name].update(config_plus)
1435                     arch_config_minus[arch][name].update(config_minus)
1436                     arch_config_change[arch][name].update(config_change)
1437
1438                     _AddConfig(lines, name, config_plus, config_minus,
1439                                config_change)
1440                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1441                            all_config_change)
1442                 summary[target] = '\n'.join(lines)
1443
1444             lines_by_target = {}
1445             for target, lines in summary.items():
1446                 if lines in lines_by_target:
1447                     lines_by_target[lines].append(target)
1448                 else:
1449                     lines_by_target[lines] = [target]
1450
1451             for arch in arch_list:
1452                 lines = []
1453                 all_plus = {}
1454                 all_minus = {}
1455                 all_change = {}
1456                 for name in self.config_filenames:
1457                     all_plus.update(arch_config_plus[arch][name])
1458                     all_minus.update(arch_config_minus[arch][name])
1459                     all_change.update(arch_config_change[arch][name])
1460                     _AddConfig(lines, name, arch_config_plus[arch][name],
1461                                arch_config_minus[arch][name],
1462                                arch_config_change[arch][name])
1463                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1464                 #arch_summary[target] = '\n'.join(lines)
1465                 if lines:
1466                     Print('%s:' % arch)
1467                     _OutputConfigInfo(lines)
1468
1469             for lines, targets in lines_by_target.items():
1470                 if not lines:
1471                     continue
1472                 Print('%s :' % ' '.join(sorted(targets)))
1473                 _OutputConfigInfo(lines.split('\n'))
1474
1475
1476         # Save our updated information for the next call to this function
1477         self._base_board_dict = board_dict
1478         self._base_err_lines = err_lines
1479         self._base_warn_lines = warn_lines
1480         self._base_err_line_boards = err_line_boards
1481         self._base_warn_line_boards = warn_line_boards
1482         self._base_config = config
1483         self._base_environment = environment
1484
1485         # Get a list of boards that did not get built, if needed
1486         not_built = []
1487         for board in board_selected:
1488             if not board in board_dict:
1489                 not_built.append(board)
1490         if not_built:
1491             Print("Boards not built (%d): %s" % (len(not_built),
1492                   ', '.join(not_built)))
1493
1494     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1495             (board_dict, err_lines, err_line_boards, warn_lines,
1496              warn_line_boards, config, environment) = self.GetResultSummary(
1497                     board_selected, commit_upto,
1498                     read_func_sizes=self._show_bloat,
1499                     read_config=self._show_config,
1500                     read_environment=self._show_environment)
1501             if commits:
1502                 msg = '%02d: %s' % (commit_upto + 1,
1503                         commits[commit_upto].subject)
1504                 Print(msg, colour=self.col.BLUE)
1505             self.PrintResultSummary(board_selected, board_dict,
1506                     err_lines if self._show_errors else [], err_line_boards,
1507                     warn_lines if self._show_errors else [], warn_line_boards,
1508                     config, environment, self._show_sizes, self._show_detail,
1509                     self._show_bloat, self._show_config, self._show_environment)
1510
1511     def ShowSummary(self, commits, board_selected):
1512         """Show a build summary for U-Boot for a given board list.
1513
1514         Reset the result summary, then repeatedly call GetResultSummary on
1515         each commit's results, then display the differences we see.
1516
1517         Args:
1518             commit: Commit objects to summarise
1519             board_selected: Dict containing boards to summarise
1520         """
1521         self.commit_count = len(commits) if commits else 1
1522         self.commits = commits
1523         self.ResetResultSummary(board_selected)
1524         self._error_lines = 0
1525
1526         for commit_upto in range(0, self.commit_count, self._step):
1527             self.ProduceResultSummary(commit_upto, commits, board_selected)
1528         if not self._error_lines:
1529             Print('(no errors to report)', colour=self.col.GREEN)
1530
1531
1532     def SetupBuild(self, board_selected, commits):
1533         """Set up ready to start a build.
1534
1535         Args:
1536             board_selected: Selected boards to build
1537             commits: Selected commits to build
1538         """
1539         # First work out how many commits we will build
1540         count = (self.commit_count + self._step - 1) // self._step
1541         self.count = len(board_selected) * count
1542         self.upto = self.warned = self.fail = 0
1543         self._timestamps = collections.deque()
1544
1545     def GetThreadDir(self, thread_num):
1546         """Get the directory path to the working dir for a thread.
1547
1548         Args:
1549             thread_num: Number of thread to check (-1 for main process, which
1550                 is treated as 0)
1551         """
1552         if self.work_in_output:
1553             return self._working_dir
1554         return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
1555
1556     def _PrepareThread(self, thread_num, setup_git):
1557         """Prepare the working directory for a thread.
1558
1559         This clones or fetches the repo into the thread's work directory.
1560         Optionally, it can create a linked working tree of the repo in the
1561         thread's work directory instead.
1562
1563         Args:
1564             thread_num: Thread number (0, 1, ...)
1565             setup_git:
1566                'clone' to set up a git clone
1567                'worktree' to set up a git worktree
1568         """
1569         thread_dir = self.GetThreadDir(thread_num)
1570         builderthread.Mkdir(thread_dir)
1571         git_dir = os.path.join(thread_dir, '.git')
1572
1573         # Create a worktree or a git repo clone for this thread if it
1574         # doesn't already exist
1575         if setup_git and self.git_dir:
1576             src_dir = os.path.abspath(self.git_dir)
1577             if os.path.isdir(git_dir):
1578                 # This is a clone of the src_dir repo, we can keep using
1579                 # it but need to fetch from src_dir.
1580                 Print('\rFetching repo for thread %d' % thread_num,
1581                       newline=False)
1582                 gitutil.Fetch(git_dir, thread_dir)
1583                 terminal.PrintClear()
1584             elif os.path.isfile(git_dir):
1585                 # This is a worktree of the src_dir repo, we don't need to
1586                 # create it again or update it in any way.
1587                 pass
1588             elif os.path.exists(git_dir):
1589                 # Don't know what could trigger this, but we probably
1590                 # can't create a git worktree/clone here.
1591                 raise ValueError('Git dir %s exists, but is not a file '
1592                                  'or a directory.' % git_dir)
1593             elif setup_git == 'worktree':
1594                 Print('\rChecking out worktree for thread %d' % thread_num,
1595                       newline=False)
1596                 gitutil.AddWorktree(src_dir, thread_dir)
1597                 terminal.PrintClear()
1598             elif setup_git == 'clone' or setup_git == True:
1599                 Print('\rCloning repo for thread %d' % thread_num,
1600                       newline=False)
1601                 gitutil.Clone(src_dir, thread_dir)
1602                 terminal.PrintClear()
1603             else:
1604                 raise ValueError("Can't setup git repo with %s." % setup_git)
1605
1606     def _PrepareWorkingSpace(self, max_threads, setup_git):
1607         """Prepare the working directory for use.
1608
1609         Set up the git repo for each thread. Creates a linked working tree
1610         if git-worktree is available, or clones the repo if it isn't.
1611
1612         Args:
1613             max_threads: Maximum number of threads we expect to need. If 0 then
1614                 1 is set up, since the main process still needs somewhere to
1615                 work
1616             setup_git: True to set up a git worktree or a git clone
1617         """
1618         builderthread.Mkdir(self._working_dir)
1619         if setup_git and self.git_dir:
1620             src_dir = os.path.abspath(self.git_dir)
1621             if gitutil.CheckWorktreeIsAvailable(src_dir):
1622                 setup_git = 'worktree'
1623                 # If we previously added a worktree but the directory for it
1624                 # got deleted, we need to prune its files from the repo so
1625                 # that we can check out another in its place.
1626                 gitutil.PruneWorktrees(src_dir)
1627             else:
1628                 setup_git = 'clone'
1629
1630         # Always do at least one thread
1631         for thread in range(max(max_threads, 1)):
1632             self._PrepareThread(thread, setup_git)
1633
1634     def _GetOutputSpaceRemovals(self):
1635         """Get the output directories ready to receive files.
1636
1637         Figure out what needs to be deleted in the output directory before it
1638         can be used. We only delete old buildman directories which have the
1639         expected name pattern. See _GetOutputDir().
1640
1641         Returns:
1642             List of full paths of directories to remove
1643         """
1644         if not self.commits:
1645             return
1646         dir_list = []
1647         for commit_upto in range(self.commit_count):
1648             dir_list.append(self._GetOutputDir(commit_upto))
1649
1650         to_remove = []
1651         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1652             if dirname not in dir_list:
1653                 leaf = dirname[len(self.base_dir) + 1:]
1654                 m =  re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
1655                 if m:
1656                     to_remove.append(dirname)
1657         return to_remove
1658
1659     def _PrepareOutputSpace(self):
1660         """Get the output directories ready to receive files.
1661
1662         We delete any output directories which look like ones we need to
1663         create. Having left over directories is confusing when the user wants
1664         to check the output manually.
1665         """
1666         to_remove = self._GetOutputSpaceRemovals()
1667         if to_remove:
1668             Print('Removing %d old build directories...' % len(to_remove),
1669                   newline=False)
1670             for dirname in to_remove:
1671                 shutil.rmtree(dirname)
1672             terminal.PrintClear()
1673
1674     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1675         """Build all commits for a list of boards
1676
1677         Args:
1678             commits: List of commits to be build, each a Commit object
1679             boards_selected: Dict of selected boards, key is target name,
1680                     value is Board object
1681             keep_outputs: True to save build output files
1682             verbose: Display build results as they are completed
1683         Returns:
1684             Tuple containing:
1685                 - number of boards that failed to build
1686                 - number of boards that issued warnings
1687                 - list of thread exceptions raised
1688         """
1689         self.commit_count = len(commits) if commits else 1
1690         self.commits = commits
1691         self._verbose = verbose
1692
1693         self.ResetResultSummary(board_selected)
1694         builderthread.Mkdir(self.base_dir, parents = True)
1695         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1696                 commits is not None)
1697         self._PrepareOutputSpace()
1698         Print('\rStarting build...', newline=False)
1699         self.SetupBuild(board_selected, commits)
1700         self.ProcessResult(None)
1701         self.thread_exceptions = []
1702         # Create jobs to build all commits for each board
1703         for brd in board_selected.values():
1704             job = builderthread.BuilderJob()
1705             job.board = brd
1706             job.commits = commits
1707             job.keep_outputs = keep_outputs
1708             job.work_in_output = self.work_in_output
1709             job.step = self._step
1710             if self.num_threads:
1711                 self.queue.put(job)
1712             else:
1713                 results = self._single_builder.RunJob(job)
1714
1715         if self.num_threads:
1716             term = threading.Thread(target=self.queue.join)
1717             term.setDaemon(True)
1718             term.start()
1719             while term.is_alive():
1720                 term.join(100)
1721
1722             # Wait until we have processed all output
1723             self.out_queue.join()
1724         Print()
1725
1726         msg = 'Completed: %d total built' % self.count
1727         if self.already_done:
1728            msg += ' (%d previously' % self.already_done
1729            if self.already_done != self.count:
1730                msg += ', %d newly' % (self.count - self.already_done)
1731            msg += ')'
1732         duration = datetime.now() - self._start_time
1733         if duration > timedelta(microseconds=1000000):
1734             if duration.microseconds >= 500000:
1735                 duration = duration + timedelta(seconds=1)
1736             duration = duration - timedelta(microseconds=duration.microseconds)
1737             rate = float(self.count) / duration.total_seconds()
1738             msg += ', duration %s, rate %1.2f' % (duration, rate)
1739         Print(msg)
1740         if self.thread_exceptions:
1741             Print('Failed: %d thread exceptions' % len(self.thread_exceptions),
1742                   colour=self.col.RED)
1743
1744         return (self.fail, self.warned, self.thread_exceptions)