tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import board
  12 from buildman import bsettings
  13 from buildman import toolchain
  14 from buildman.builder import Builder
  15 from patman import command
  16 from patman import gitutil
  17 from patman import patchstream
  18 from patman import terminal
  19 from patman import tools
  20 from patman.terminal import Print
  21
  22 def GetPlural(count):
  23     """Returns a plural 's' if count is not 1"""
  24     return 's' if count != 1 else ''
  25
  26 def GetActionSummary(is_summary, commits, selected, options):
  27     """Return a string summarising the intended action.
  28
  29     Returns:
  30         Summary string.
  31     """
  32     if commits:
  33         count = len(commits)
  34         count = (count + options.step - 1) // options.step
  35         commit_str = '%d commit%s' % (count, GetPlural(count))
  36     else:
  37         commit_str = 'current source'
  38     str = '%s %s for %d boards' % (
  39         'Summary of' if is_summary else 'Building', commit_str,
  40         len(selected))
  41     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  42             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  43     return str
  44
  45 def ShowActions(series, why_selected, boards_selected, builder, options,
  46                 board_warnings):
  47     """Display a list of actions that we would take, if not a dry run.
  48
  49     Args:
  50         series: Series object
  51         why_selected: Dictionary where each key is a buildman argument
  52                 provided by the user, and the value is the list of boards
  53                 brought in by that argument. For example, 'arm' might bring
  54                 in 400 boards, so in this case the key would be 'arm' and
  55                 the value would be a list of board names.
  56         boards_selected: Dict of selected boards, key is target name,
  57                 value is Board object
  58         builder: The builder that will be used to build the commits
  59         options: Command line options object
  60         board_warnings: List of warnings obtained from board selected
  61     """
  62     col = terminal.Color()
  63     print('Dry run, so not doing much. But I would do this:')
  64     print()
  65     if series:
  66         commits = series.commits
  67     else:
  68         commits = None
  69     print(GetActionSummary(False, commits, boards_selected,
  70             options))
  71     print('Build directory: %s' % builder.base_dir)
  72     if commits:
  73         for upto in range(0, len(series.commits), options.step):
  74             commit = series.commits[upto]
  75             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  76             print(commit.subject)
  77     print()
  78     for arg in why_selected:
  79         if arg != 'all':
  80             print(arg, ': %d boards' % len(why_selected[arg]))
  81             if options.verbose:
  82                 print('   %s' % ' '.join(why_selected[arg]))
  83     print(('Total boards to build for each commit: %d\n' %
  84             len(why_selected['all'])))
  85     if board_warnings:
  86         for warning in board_warnings:
  87             print(col.Color(col.YELLOW, warning))
  88
  89 def ShowToolchainPrefix(boards, toolchains):
  90     """Show information about a the tool chain used by one or more boards
  91
  92     The function checks that all boards use the same toolchain, then prints
  93     the correct value for CROSS_COMPILE.
  94
  95     Args:
  96         boards: Boards object containing selected boards
  97         toolchains: Toolchains object containing available toolchains
  98
  99     Return:
 100         None on success, string error message otherwise
 101     """
 102     boards = boards.GetSelectedDict()
 103     tc_set = set()
 104     for brd in boards.values():
 105         tc_set.add(toolchains.Select(brd.arch))
 106     if len(tc_set) != 1:
 107         return 'Supplied boards must share one toolchain'
 108         return False
 109     tc = tc_set.pop()
 110     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 111     return None
 112
 113 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 114                clean_dir=False, test_thread_exceptions=False):
 115     """The main control code for buildman
 116
 117     Args:
 118         options: Command line options object
 119         args: Command line arguments (list of strings)
 120         toolchains: Toolchains to use - this should be a Toolchains()
 121                 object. If None, then it will be created and scanned
 122         make_func: Make function to use for the builder. This is called
 123                 to execute 'make'. If this is None, the normal function
 124                 will be used, which calls the 'make' tool with suitable
 125                 arguments. This setting is useful for tests.
 126         board: Boards() object to use, containing a list of available
 127                 boards. If this is None it will be created and scanned.
 128         clean_dir: Used for tests only, indicates that the existing output_dir
 129             should be removed before starting the build
 130         test_thread_exceptions: Uses for tests only, True to make the threads
 131             raise an exception instead of reporting their result. This simulates
 132             a failure in the code somewhere
 133     """
 134     global builder
 135
 136     if options.full_help:
 137         tools.PrintFullHelp(
 138             os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README')
 139         )
 140         return 0
 141
 142     gitutil.Setup()
 143     col = terminal.Color()
 144
 145     options.git_dir = os.path.join(options.git, '.git')
 146
 147     no_toolchains = toolchains is None
 148     if no_toolchains:
 149         toolchains = toolchain.Toolchains(options.override_toolchain)
 150
 151     if options.fetch_arch:
 152         if options.fetch_arch == 'list':
 153             sorted_list = toolchains.ListArchs()
 154             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 155                             ' '.join(sorted_list)))
 156             return 0
 157         else:
 158             fetch_arch = options.fetch_arch
 159             if fetch_arch == 'all':
 160                 fetch_arch = ','.join(toolchains.ListArchs())
 161                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 162                                 fetch_arch))
 163             for arch in fetch_arch.split(','):
 164                 print()
 165                 ret = toolchains.FetchAndInstall(arch)
 166                 if ret:
 167                     return ret
 168             return 0
 169
 170     if no_toolchains:
 171         toolchains.GetSettings()
 172         toolchains.Scan(options.list_tool_chains and options.verbose)
 173     if options.list_tool_chains:
 174         toolchains.List()
 175         print()
 176         return 0
 177
 178     if options.incremental:
 179         print(col.Color(col.RED,
 180                         'Warning: -I has been removed. See documentation'))
 181     if not options.output_dir:
 182         if options.work_in_output:
 183             sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
 184         options.output_dir = '..'
 185
 186     # Work out what subset of the boards we are building
 187     if not boards:
 188         if not os.path.exists(options.output_dir):
 189             os.makedirs(options.output_dir)
 190         board_file = os.path.join(options.output_dir, 'boards.cfg')
 191         our_path = os.path.dirname(os.path.realpath(__file__))
 192         genboardscfg = os.path.join(our_path, '../genboardscfg.py')
 193         if not os.path.exists(genboardscfg):
 194             genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 195         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 196         if status != 0:
 197             # Older versions don't support -q
 198             status = subprocess.call([genboardscfg, '-o', board_file])
 199             if status != 0:
 200                 sys.exit("Failed to generate boards.cfg")
 201
 202         boards = board.Boards()
 203         boards.ReadBoards(board_file)
 204
 205     exclude = []
 206     if options.exclude:
 207         for arg in options.exclude:
 208             exclude += arg.split(',')
 209
 210     if options.boards:
 211         requested_boards = []
 212         for b in options.boards:
 213             requested_boards += b.split(',')
 214     else:
 215         requested_boards = None
 216     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 217                                                        requested_boards)
 218     selected = boards.GetSelected()
 219     if not len(selected):
 220         sys.exit(col.Color(col.RED, 'No matching boards found'))
 221
 222     if options.print_prefix:
 223         err = ShowToolchainPrefix(boards, toolchains)
 224         if err:
 225             sys.exit(col.Color(col.RED, err))
 226         return 0
 227
 228     # Work out how many commits to build. We want to build everything on the
 229     # branch. We also build the upstream commit as a control so we can see
 230     # problems introduced by the first commit on the branch.
 231     count = options.count
 232     has_range = options.branch and '..' in options.branch
 233     if count == -1:
 234         if not options.branch:
 235             count = 1
 236         else:
 237             if has_range:
 238                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 239                                                          options.branch)
 240             else:
 241                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 242                                                           options.branch)
 243             if count is None:
 244                 sys.exit(col.Color(col.RED, msg))
 245             elif count == 0:
 246                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 247                                    options.branch))
 248             if msg:
 249                 print(col.Color(col.YELLOW, msg))
 250             count += 1   # Build upstream commit also
 251
 252     if not count:
 253         str = ("No commits found to process in branch '%s': "
 254                "set branch's upstream or use -c flag" % options.branch)
 255         sys.exit(col.Color(col.RED, str))
 256     if options.work_in_output:
 257         if len(selected) != 1:
 258             sys.exit(col.Color(col.RED,
 259                                '-w can only be used with a single board'))
 260         if count != 1:
 261             sys.exit(col.Color(col.RED,
 262                                '-w can only be used with a single commit'))
 263
 264     # Read the metadata from the commits. First look at the upstream commit,
 265     # then the ones in the branch. We would like to do something like
 266     # upstream/master~..branch but that isn't possible if upstream/master is
 267     # a merge commit (it will list all the commits that form part of the
 268     # merge)
 269     # Conflicting tags are not a problem for buildman, since it does not use
 270     # them. For example, Series-version is not useful for buildman. On the
 271     # other hand conflicting tags will cause an error. So allow later tags
 272     # to overwrite earlier ones by setting allow_overwrite=True
 273     if options.branch:
 274         if count == -1:
 275             if has_range:
 276                 range_expr = options.branch
 277             else:
 278                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 279                                                       options.branch)
 280             upstream_commit = gitutil.GetUpstream(options.git_dir,
 281                                                   options.branch)
 282             series = patchstream.get_metadata_for_list(upstream_commit,
 283                 options.git_dir, 1, series=None, allow_overwrite=True)
 284
 285             series = patchstream.get_metadata_for_list(range_expr,
 286                     options.git_dir, None, series, allow_overwrite=True)
 287         else:
 288             # Honour the count
 289             series = patchstream.get_metadata_for_list(options.branch,
 290                     options.git_dir, count, series=None, allow_overwrite=True)
 291     else:
 292         series = None
 293         if not options.dry_run:
 294             options.verbose = True
 295             if not options.summary:
 296                 options.show_errors = True
 297
 298     # By default we have one thread per CPU. But if there are not enough jobs
 299     # we can have fewer threads and use a high '-j' value for make.
 300     if options.threads is None:
 301         options.threads = min(multiprocessing.cpu_count(), len(selected))
 302     if not options.jobs:
 303         options.jobs = max(1, (multiprocessing.cpu_count() +
 304                 len(selected) - 1) // len(selected))
 305
 306     if not options.step:
 307         options.step = len(series.commits) - 1
 308
 309     gnu_make = command.Output(os.path.join(options.git,
 310             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 311     if not gnu_make:
 312         sys.exit('GNU Make not found')
 313
 314     # Create a new builder with the selected options.
 315     output_dir = options.output_dir
 316     if options.branch:
 317         dirname = options.branch.replace('/', '_')
 318         # As a special case allow the board directory to be placed in the
 319         # output directory itself rather than any subdirectory.
 320         if not options.no_subdirs:
 321             output_dir = os.path.join(options.output_dir, dirname)
 322         if clean_dir and os.path.exists(output_dir):
 323             shutil.rmtree(output_dir)
 324     builder = Builder(toolchains, output_dir, options.git_dir,
 325             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 326             show_unknown=options.show_unknown, step=options.step,
 327             no_subdirs=options.no_subdirs, full_path=options.full_path,
 328             verbose_build=options.verbose_build,
 329             mrproper=options.mrproper,
 330             per_board_out_dir=options.per_board_out_dir,
 331             config_only=options.config_only,
 332             squash_config_y=not options.preserve_config_y,
 333             warnings_as_errors=options.warnings_as_errors,
 334             work_in_output=options.work_in_output,
 335             test_thread_exceptions=test_thread_exceptions)
 336     builder.force_config_on_failure = not options.quick
 337     if make_func:
 338         builder.do_make = make_func
 339
 340     # For a dry run, just show our actions as a sanity check
 341     if options.dry_run:
 342         ShowActions(series, why_selected, selected, builder, options,
 343                     board_warnings)
 344     else:
 345         builder.force_build = options.force_build
 346         builder.force_build_failures = options.force_build_failures
 347         builder.force_reconfig = options.force_reconfig
 348         builder.in_tree = options.in_tree
 349
 350         # Work out which boards to build
 351         board_selected = boards.GetSelectedDict()
 352
 353         if series:
 354             commits = series.commits
 355             # Number the commits for test purposes
 356             for commit in range(len(commits)):
 357                 commits[commit].sequence = commit
 358         else:
 359             commits = None
 360
 361         Print(GetActionSummary(options.summary, commits, board_selected,
 362                                options))
 363
 364         # We can't show function sizes without board details at present
 365         if options.show_bloat:
 366             options.show_detail = True
 367         builder.SetDisplayOptions(
 368             options.show_errors, options.show_sizes, options.show_detail,
 369             options.show_bloat, options.list_error_boards, options.show_config,
 370             options.show_environment, options.filter_dtb_warnings,
 371             options.filter_migration_warnings)
 372         if options.summary:
 373             builder.ShowSummary(commits, board_selected)
 374         else:
 375             fail, warned, excs = builder.BuildBoards(
 376                 commits, board_selected, options.keep_outputs, options.verbose)
 377             if excs:
 378                 return 102
 379             elif fail:
 380                 return 100
 381             elif warned and not options.ignore_warnings:
 382                 return 101
 383     return 0