tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 from terminal import Print
  17 import toolchain
  18 import command
  19 import subprocess
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def ShowToolchainPrefix(boards, toolchains):
  89     """Show information about a the tool chain used by one or more boards
  90
  91     The function checks that all boards use the same toolchain, then prints
  92     the correct value for CROSS_COMPILE.
  93
  94     Args:
  95         boards: Boards object containing selected boards
  96         toolchains: Toolchains object containing available toolchains
  97
  98     Return:
  99         None on success, string error message otherwise
 100     """
 101     boards = boards.GetSelectedDict()
 102     tc_set = set()
 103     for brd in boards.values():
 104         tc_set.add(toolchains.Select(brd.arch))
 105     if len(tc_set) != 1:
 106         return 'Supplied boards must share one toolchain'
 107         return False
 108     tc = tc_set.pop()
 109     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 110     return None
 111
 112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 113                clean_dir=False):
 114     """The main control code for buildman
 115
 116     Args:
 117         options: Command line options object
 118         args: Command line arguments (list of strings)
 119         toolchains: Toolchains to use - this should be a Toolchains()
 120                 object. If None, then it will be created and scanned
 121         make_func: Make function to use for the builder. This is called
 122                 to execute 'make'. If this is None, the normal function
 123                 will be used, which calls the 'make' tool with suitable
 124                 arguments. This setting is useful for tests.
 125         board: Boards() object to use, containing a list of available
 126                 boards. If this is None it will be created and scanned.
 127     """
 128     global builder
 129
 130     if options.full_help:
 131         pager = os.getenv('PAGER')
 132         if not pager:
 133             pager = 'more'
 134         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 135                              'README')
 136         command.Run(pager, fname)
 137         return 0
 138
 139     gitutil.Setup()
 140     col = terminal.Color()
 141
 142     options.git_dir = os.path.join(options.git, '.git')
 143
 144     no_toolchains = toolchains is None
 145     if no_toolchains:
 146         toolchains = toolchain.Toolchains(options.override_toolchain)
 147
 148     if options.fetch_arch:
 149         if options.fetch_arch == 'list':
 150             sorted_list = toolchains.ListArchs()
 151             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 152                             ' '.join(sorted_list)))
 153             return 0
 154         else:
 155             fetch_arch = options.fetch_arch
 156             if fetch_arch == 'all':
 157                 fetch_arch = ','.join(toolchains.ListArchs())
 158                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 159                                 fetch_arch))
 160             for arch in fetch_arch.split(','):
 161                 print()
 162                 ret = toolchains.FetchAndInstall(arch)
 163                 if ret:
 164                     return ret
 165             return 0
 166
 167     if no_toolchains:
 168         toolchains.GetSettings()
 169         toolchains.Scan(options.list_tool_chains and options.verbose)
 170     if options.list_tool_chains:
 171         toolchains.List()
 172         print()
 173         return 0
 174
 175     if options.incremental:
 176         print(col.Color(col.RED,
 177                         'Warning: -I has been removed. See documentation'))
 178
 179     # Work out what subset of the boards we are building
 180     if not boards:
 181         if not os.path.exists(options.output_dir):
 182             os.makedirs(options.output_dir)
 183         board_file = os.path.join(options.output_dir, 'boards.cfg')
 184         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 185         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 186         if status != 0:
 187             sys.exit("Failed to generate boards.cfg")
 188
 189         boards = board.Boards()
 190         boards.ReadBoards(board_file)
 191
 192     exclude = []
 193     if options.exclude:
 194         for arg in options.exclude:
 195             exclude += arg.split(',')
 196
 197     if options.boards:
 198         requested_boards = []
 199         for b in options.boards:
 200             requested_boards += b.split(',')
 201     else:
 202         requested_boards = None
 203     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 204                                                        requested_boards)
 205     selected = boards.GetSelected()
 206     if not len(selected):
 207         sys.exit(col.Color(col.RED, 'No matching boards found'))
 208
 209     if options.print_prefix:
 210         err = ShowToolchainInfo(boards, toolchains)
 211         if err:
 212             sys.exit(col.Color(col.RED, err))
 213         return 0
 214
 215     # Work out how many commits to build. We want to build everything on the
 216     # branch. We also build the upstream commit as a control so we can see
 217     # problems introduced by the first commit on the branch.
 218     count = options.count
 219     has_range = options.branch and '..' in options.branch
 220     if count == -1:
 221         if not options.branch:
 222             count = 1
 223         else:
 224             if has_range:
 225                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 226                                                          options.branch)
 227             else:
 228                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 229                                                           options.branch)
 230             if count is None:
 231                 sys.exit(col.Color(col.RED, msg))
 232             elif count == 0:
 233                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 234                                    options.branch))
 235             if msg:
 236                 print(col.Color(col.YELLOW, msg))
 237             count += 1   # Build upstream commit also
 238
 239     if not count:
 240         str = ("No commits found to process in branch '%s': "
 241                "set branch's upstream or use -c flag" % options.branch)
 242         sys.exit(col.Color(col.RED, str))
 243     if options.work_in_output:
 244         if len(selected) != 1:
 245             sys.exit(col.Color(col.RED,
 246                                '-w can only be used with a single board'))
 247         if count != 1:
 248             sys.exit(col.Color(col.RED,
 249                                '-w can only be used with a single commit'))
 250
 251     # Read the metadata from the commits. First look at the upstream commit,
 252     # then the ones in the branch. We would like to do something like
 253     # upstream/master~..branch but that isn't possible if upstream/master is
 254     # a merge commit (it will list all the commits that form part of the
 255     # merge)
 256     # Conflicting tags are not a problem for buildman, since it does not use
 257     # them. For example, Series-version is not useful for buildman. On the
 258     # other hand conflicting tags will cause an error. So allow later tags
 259     # to overwrite earlier ones by setting allow_overwrite=True
 260     if options.branch:
 261         if count == -1:
 262             if has_range:
 263                 range_expr = options.branch
 264             else:
 265                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 266                                                       options.branch)
 267             upstream_commit = gitutil.GetUpstream(options.git_dir,
 268                                                   options.branch)
 269             series = patchstream.GetMetaDataForList(upstream_commit,
 270                 options.git_dir, 1, series=None, allow_overwrite=True)
 271
 272             series = patchstream.GetMetaDataForList(range_expr,
 273                     options.git_dir, None, series, allow_overwrite=True)
 274         else:
 275             # Honour the count
 276             series = patchstream.GetMetaDataForList(options.branch,
 277                     options.git_dir, count, series=None, allow_overwrite=True)
 278     else:
 279         series = None
 280         if not options.dry_run:
 281             options.verbose = True
 282             if not options.summary:
 283                 options.show_errors = True
 284
 285     # By default we have one thread per CPU. But if there are not enough jobs
 286     # we can have fewer threads and use a high '-j' value for make.
 287     if not options.threads:
 288         options.threads = min(multiprocessing.cpu_count(), len(selected))
 289     if not options.jobs:
 290         options.jobs = max(1, (multiprocessing.cpu_count() +
 291                 len(selected) - 1) // len(selected))
 292
 293     if not options.step:
 294         options.step = len(series.commits) - 1
 295
 296     gnu_make = command.Output(os.path.join(options.git,
 297             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 298     if not gnu_make:
 299         sys.exit('GNU Make not found')
 300
 301     # Create a new builder with the selected options.
 302     output_dir = options.output_dir
 303     if options.branch:
 304         dirname = options.branch.replace('/', '_')
 305         # As a special case allow the board directory to be placed in the
 306         # output directory itself rather than any subdirectory.
 307         if not options.no_subdirs:
 308             output_dir = os.path.join(options.output_dir, dirname)
 309         if clean_dir and os.path.exists(output_dir):
 310             shutil.rmtree(output_dir)
 311     builder = Builder(toolchains, output_dir, options.git_dir,
 312             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 313             show_unknown=options.show_unknown, step=options.step,
 314             no_subdirs=options.no_subdirs, full_path=options.full_path,
 315             verbose_build=options.verbose_build,
 316             mrproper=options.mrproper,
 317             per_board_out_dir=options.per_board_out_dir,
 318             config_only=options.config_only,
 319             squash_config_y=not options.preserve_config_y,
 320             warnings_as_errors=options.warnings_as_errors,
 321             work_in_output=options.work_in_output)
 322     builder.force_config_on_failure = not options.quick
 323     if make_func:
 324         builder.do_make = make_func
 325
 326     # For a dry run, just show our actions as a sanity check
 327     if options.dry_run:
 328         ShowActions(series, why_selected, selected, builder, options,
 329                     board_warnings)
 330     else:
 331         builder.force_build = options.force_build
 332         builder.force_build_failures = options.force_build_failures
 333         builder.force_reconfig = options.force_reconfig
 334         builder.in_tree = options.in_tree
 335
 336         # Work out which boards to build
 337         board_selected = boards.GetSelectedDict()
 338
 339         if series:
 340             commits = series.commits
 341             # Number the commits for test purposes
 342             for commit in range(len(commits)):
 343                 commits[commit].sequence = commit
 344         else:
 345             commits = None
 346
 347         Print(GetActionSummary(options.summary, commits, board_selected,
 348                                options))
 349
 350         # We can't show function sizes without board details at present
 351         if options.show_bloat:
 352             options.show_detail = True
 353         builder.SetDisplayOptions(
 354             options.show_errors, options.show_sizes, options.show_detail,
 355             options.show_bloat, options.list_error_boards, options.show_config,
 356             options.show_environment, options.filter_dtb_warnings,
 357             options.filter_migration_warnings)
 358         if options.summary:
 359             builder.ShowSummary(commits, board_selected)
 360         else:
 361             fail, warned = builder.BuildBoards(commits, board_selected,
 362                                 options.keep_outputs, options.verbose)
 363             if fail:
 364                 return 100
 365             elif warned and not options.ignore_warnings:
 366                 return 101
 367     return 0