tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import board
  12 from buildman import bsettings
  13 from buildman import cfgutil
  14 from buildman import toolchain
  15 from buildman.builder import Builder
  16 from patman import command
  17 from patman import gitutil
  18 from patman import patchstream
  19 from patman import terminal
  20 from patman import tools
  21 from patman.terminal import Print
  22
  23 def GetPlural(count):
  24     """Returns a plural 's' if count is not 1"""
  25     return 's' if count != 1 else ''
  26
  27 def GetActionSummary(is_summary, commits, selected, options):
  28     """Return a string summarising the intended action.
  29
  30     Returns:
  31         Summary string.
  32     """
  33     if commits:
  34         count = len(commits)
  35         count = (count + options.step - 1) // options.step
  36         commit_str = '%d commit%s' % (count, GetPlural(count))
  37     else:
  38         commit_str = 'current source'
  39     str = '%s %s for %d boards' % (
  40         'Summary of' if is_summary else 'Building', commit_str,
  41         len(selected))
  42     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  43             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  44     return str
  45
  46 def ShowActions(series, why_selected, boards_selected, builder, options,
  47                 board_warnings):
  48     """Display a list of actions that we would take, if not a dry run.
  49
  50     Args:
  51         series: Series object
  52         why_selected: Dictionary where each key is a buildman argument
  53                 provided by the user, and the value is the list of boards
  54                 brought in by that argument. For example, 'arm' might bring
  55                 in 400 boards, so in this case the key would be 'arm' and
  56                 the value would be a list of board names.
  57         boards_selected: Dict of selected boards, key is target name,
  58                 value is Board object
  59         builder: The builder that will be used to build the commits
  60         options: Command line options object
  61         board_warnings: List of warnings obtained from board selected
  62     """
  63     col = terminal.Color()
  64     print('Dry run, so not doing much. But I would do this:')
  65     print()
  66     if series:
  67         commits = series.commits
  68     else:
  69         commits = None
  70     print(GetActionSummary(False, commits, boards_selected,
  71             options))
  72     print('Build directory: %s' % builder.base_dir)
  73     if commits:
  74         for upto in range(0, len(series.commits), options.step):
  75             commit = series.commits[upto]
  76             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  77             print(commit.subject)
  78     print()
  79     for arg in why_selected:
  80         if arg != 'all':
  81             print(arg, ': %d boards' % len(why_selected[arg]))
  82             if options.verbose:
  83                 print('   %s' % ' '.join(why_selected[arg]))
  84     print(('Total boards to build for each commit: %d\n' %
  85             len(why_selected['all'])))
  86     if board_warnings:
  87         for warning in board_warnings:
  88             print(col.Color(col.YELLOW, warning))
  89
  90 def ShowToolchainPrefix(boards, toolchains):
  91     """Show information about a the tool chain used by one or more boards
  92
  93     The function checks that all boards use the same toolchain, then prints
  94     the correct value for CROSS_COMPILE.
  95
  96     Args:
  97         boards: Boards object containing selected boards
  98         toolchains: Toolchains object containing available toolchains
  99
 100     Return:
 101         None on success, string error message otherwise
 102     """
 103     boards = boards.GetSelectedDict()
 104     tc_set = set()
 105     for brd in boards.values():
 106         tc_set.add(toolchains.Select(brd.arch))
 107     if len(tc_set) != 1:
 108         return 'Supplied boards must share one toolchain'
 109         return False
 110     tc = tc_set.pop()
 111     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 112     return None
 113
 114 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 115                clean_dir=False, test_thread_exceptions=False):
 116     """The main control code for buildman
 117
 118     Args:
 119         options: Command line options object
 120         args: Command line arguments (list of strings)
 121         toolchains: Toolchains to use - this should be a Toolchains()
 122                 object. If None, then it will be created and scanned
 123         make_func: Make function to use for the builder. This is called
 124                 to execute 'make'. If this is None, the normal function
 125                 will be used, which calls the 'make' tool with suitable
 126                 arguments. This setting is useful for tests.
 127         board: Boards() object to use, containing a list of available
 128                 boards. If this is None it will be created and scanned.
 129         clean_dir: Used for tests only, indicates that the existing output_dir
 130             should be removed before starting the build
 131         test_thread_exceptions: Uses for tests only, True to make the threads
 132             raise an exception instead of reporting their result. This simulates
 133             a failure in the code somewhere
 134     """
 135     global builder
 136
 137     if options.full_help:
 138         tools.PrintFullHelp(
 139             os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README')
 140         )
 141         return 0
 142
 143     gitutil.Setup()
 144     col = terminal.Color()
 145
 146     options.git_dir = os.path.join(options.git, '.git')
 147
 148     no_toolchains = toolchains is None
 149     if no_toolchains:
 150         toolchains = toolchain.Toolchains(options.override_toolchain)
 151
 152     if options.fetch_arch:
 153         if options.fetch_arch == 'list':
 154             sorted_list = toolchains.ListArchs()
 155             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 156                             ' '.join(sorted_list)))
 157             return 0
 158         else:
 159             fetch_arch = options.fetch_arch
 160             if fetch_arch == 'all':
 161                 fetch_arch = ','.join(toolchains.ListArchs())
 162                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 163                                 fetch_arch))
 164             for arch in fetch_arch.split(','):
 165                 print()
 166                 ret = toolchains.FetchAndInstall(arch)
 167                 if ret:
 168                     return ret
 169             return 0
 170
 171     if no_toolchains:
 172         toolchains.GetSettings()
 173         toolchains.Scan(options.list_tool_chains and options.verbose)
 174     if options.list_tool_chains:
 175         toolchains.List()
 176         print()
 177         return 0
 178
 179     if options.incremental:
 180         print(col.Color(col.RED,
 181                         'Warning: -I has been removed. See documentation'))
 182     if not options.output_dir:
 183         if options.work_in_output:
 184             sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
 185         options.output_dir = '..'
 186
 187     # Work out what subset of the boards we are building
 188     if not boards:
 189         if not os.path.exists(options.output_dir):
 190             os.makedirs(options.output_dir)
 191         board_file = os.path.join(options.output_dir, 'boards.cfg')
 192         our_path = os.path.dirname(os.path.realpath(__file__))
 193         genboardscfg = os.path.join(our_path, '../genboardscfg.py')
 194         if not os.path.exists(genboardscfg):
 195             genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 196         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 197         if status != 0:
 198             # Older versions don't support -q
 199             status = subprocess.call([genboardscfg, '-o', board_file])
 200             if status != 0:
 201                 sys.exit("Failed to generate boards.cfg")
 202
 203         boards = board.Boards()
 204         boards.ReadBoards(board_file)
 205
 206     exclude = []
 207     if options.exclude:
 208         for arg in options.exclude:
 209             exclude += arg.split(',')
 210
 211     if options.boards:
 212         requested_boards = []
 213         for b in options.boards:
 214             requested_boards += b.split(',')
 215     else:
 216         requested_boards = None
 217     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 218                                                        requested_boards)
 219     selected = boards.GetSelected()
 220     if not len(selected):
 221         sys.exit(col.Color(col.RED, 'No matching boards found'))
 222
 223     if options.print_prefix:
 224         err = ShowToolchainPrefix(boards, toolchains)
 225         if err:
 226             sys.exit(col.Color(col.RED, err))
 227         return 0
 228
 229     # Work out how many commits to build. We want to build everything on the
 230     # branch. We also build the upstream commit as a control so we can see
 231     # problems introduced by the first commit on the branch.
 232     count = options.count
 233     has_range = options.branch and '..' in options.branch
 234     if count == -1:
 235         if not options.branch:
 236             count = 1
 237         else:
 238             if has_range:
 239                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 240                                                          options.branch)
 241             else:
 242                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 243                                                           options.branch)
 244             if count is None:
 245                 sys.exit(col.Color(col.RED, msg))
 246             elif count == 0:
 247                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 248                                    options.branch))
 249             if msg:
 250                 print(col.Color(col.YELLOW, msg))
 251             count += 1   # Build upstream commit also
 252
 253     if not count:
 254         str = ("No commits found to process in branch '%s': "
 255                "set branch's upstream or use -c flag" % options.branch)
 256         sys.exit(col.Color(col.RED, str))
 257     if options.work_in_output:
 258         if len(selected) != 1:
 259             sys.exit(col.Color(col.RED,
 260                                '-w can only be used with a single board'))
 261         if count != 1:
 262             sys.exit(col.Color(col.RED,
 263                                '-w can only be used with a single commit'))
 264
 265     # Read the metadata from the commits. First look at the upstream commit,
 266     # then the ones in the branch. We would like to do something like
 267     # upstream/master~..branch but that isn't possible if upstream/master is
 268     # a merge commit (it will list all the commits that form part of the
 269     # merge)
 270     # Conflicting tags are not a problem for buildman, since it does not use
 271     # them. For example, Series-version is not useful for buildman. On the
 272     # other hand conflicting tags will cause an error. So allow later tags
 273     # to overwrite earlier ones by setting allow_overwrite=True
 274     if options.branch:
 275         if count == -1:
 276             if has_range:
 277                 range_expr = options.branch
 278             else:
 279                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 280                                                       options.branch)
 281             upstream_commit = gitutil.GetUpstream(options.git_dir,
 282                                                   options.branch)
 283             series = patchstream.get_metadata_for_list(upstream_commit,
 284                 options.git_dir, 1, series=None, allow_overwrite=True)
 285
 286             series = patchstream.get_metadata_for_list(range_expr,
 287                     options.git_dir, None, series, allow_overwrite=True)
 288         else:
 289             # Honour the count
 290             series = patchstream.get_metadata_for_list(options.branch,
 291                     options.git_dir, count, series=None, allow_overwrite=True)
 292     else:
 293         series = None
 294         if not options.dry_run:
 295             options.verbose = True
 296             if not options.summary:
 297                 options.show_errors = True
 298
 299     # By default we have one thread per CPU. But if there are not enough jobs
 300     # we can have fewer threads and use a high '-j' value for make.
 301     if options.threads is None:
 302         options.threads = min(multiprocessing.cpu_count(), len(selected))
 303     if not options.jobs:
 304         options.jobs = max(1, (multiprocessing.cpu_count() +
 305                 len(selected) - 1) // len(selected))
 306
 307     if not options.step:
 308         options.step = len(series.commits) - 1
 309
 310     gnu_make = command.Output(os.path.join(options.git,
 311             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 312     if not gnu_make:
 313         sys.exit('GNU Make not found')
 314
 315     # Create a new builder with the selected options.
 316     output_dir = options.output_dir
 317     if options.branch:
 318         dirname = options.branch.replace('/', '_')
 319         # As a special case allow the board directory to be placed in the
 320         # output directory itself rather than any subdirectory.
 321         if not options.no_subdirs:
 322             output_dir = os.path.join(options.output_dir, dirname)
 323         if clean_dir and os.path.exists(output_dir):
 324             shutil.rmtree(output_dir)
 325     adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg)
 326
 327     builder = Builder(toolchains, output_dir, options.git_dir,
 328             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 329             show_unknown=options.show_unknown, step=options.step,
 330             no_subdirs=options.no_subdirs, full_path=options.full_path,
 331             verbose_build=options.verbose_build,
 332             mrproper=options.mrproper,
 333             per_board_out_dir=options.per_board_out_dir,
 334             config_only=options.config_only,
 335             squash_config_y=not options.preserve_config_y,
 336             warnings_as_errors=options.warnings_as_errors,
 337             work_in_output=options.work_in_output,
 338             test_thread_exceptions=test_thread_exceptions,
 339             adjust_cfg=adjust_cfg)
 340     builder.force_config_on_failure = not options.quick
 341     if make_func:
 342         builder.do_make = make_func
 343
 344     # For a dry run, just show our actions as a sanity check
 345     if options.dry_run:
 346         ShowActions(series, why_selected, selected, builder, options,
 347                     board_warnings)
 348     else:
 349         builder.force_build = options.force_build
 350         builder.force_build_failures = options.force_build_failures
 351         builder.force_reconfig = options.force_reconfig
 352         builder.in_tree = options.in_tree
 353
 354         # Work out which boards to build
 355         board_selected = boards.GetSelectedDict()
 356
 357         if series:
 358             commits = series.commits
 359             # Number the commits for test purposes
 360             for commit in range(len(commits)):
 361                 commits[commit].sequence = commit
 362         else:
 363             commits = None
 364
 365         Print(GetActionSummary(options.summary, commits, board_selected,
 366                                options))
 367
 368         # We can't show function sizes without board details at present
 369         if options.show_bloat:
 370             options.show_detail = True
 371         builder.SetDisplayOptions(
 372             options.show_errors, options.show_sizes, options.show_detail,
 373             options.show_bloat, options.list_error_boards, options.show_config,
 374             options.show_environment, options.filter_dtb_warnings,
 375             options.filter_migration_warnings)
 376         if options.summary:
 377             builder.ShowSummary(commits, board_selected)
 378         else:
 379             fail, warned, excs = builder.BuildBoards(
 380                 commits, board_selected, options.keep_outputs, options.verbose)
 381             if excs:
 382                 return 102
 383             elif fail:
 384                 return 100
 385             elif warned and not options.ignore_warnings:
 386                 return 101
 387     return 0