import sys
import threading
-import command
-import gitutil
+from buildman import cfgutil
+from patman import command
+from patman import gitutil
RETURN_CODE_RETRY = -1
+BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl']
def Mkdir(dirname, parents = False):
"""Make a directory if it doesn't already exist.
except OSError as err:
if err.errno == errno.EEXIST:
if os.path.realpath('.') == os.path.realpath(dirname):
- print "Cannot create the current working directory '%s'!" % dirname
+ print("Cannot create the current working directory '%s'!" % dirname)
sys.exit(1)
pass
else:
Members:
board: Board object to build
- commits: List of commit options to build.
+ commits: List of Commit objects to build
+ keep_outputs: True to save build output files
+ step: 1 to process every commit, n to process every nth commit
+ work_in_output: Use the output directory as the work directory and
+ don't write to a separate output directory.
"""
def __init__(self):
self.board = None
self.commits = []
+ self.keep_outputs = False
+ self.step = 1
+ self.work_in_output = False
class ResultThread(threading.Thread):
Members:
builder: The builder which contains information we might need
thread_num: Our thread number (0-n-1), used to decide on a
- temporary directory
+ temporary directory. If this is -1 then there are no threads
+ and we are the (only) main process
+ mrproper: Use 'make mrproper' before each reconfigure
+ per_board_out_dir: True to build in a separate persistent directory per
+ board rather than a thread-specific directory
+ test_exception: Used for testing; True to raise an exception instead of
+ reporting the build result
"""
- def __init__(self, builder, thread_num, incremental, per_board_out_dir):
+ def __init__(self, builder, thread_num, mrproper, per_board_out_dir,
+ test_exception=False):
"""Set up a new builder thread"""
threading.Thread.__init__(self)
self.builder = builder
self.thread_num = thread_num
- self.incremental = incremental
+ self.mrproper = mrproper
self.per_board_out_dir = per_board_out_dir
+ self.test_exception = test_exception
def Make(self, commit, brd, stage, cwd, *args, **kwargs):
"""Run 'make' on a particular commit and board.
**kwargs)
def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
- force_build, force_build_failures):
+ force_build, force_build_failures, work_in_output,
+ adjust_cfg):
"""Build a particular commit.
If the build is already done, and we are not forcing a build, we skip
force_build: Force a build even if one was previously done
force_build_failures: Force a bulid if the previous result showed
failure
+ work_in_output: Use the output directory as the work directory and
+ don't write to a separate output directory.
+ adjust_cfg (list of str): List of changes to make to .config file
+ before building. Each is one of (where C is either CONFIG_xxx
+ or just xxx):
+ C to enable C
+ ~C to disable C
+ C=val to set the value of C (val must have quotes if C is
+ a string Kconfig
Returns:
tuple containing:
# self.Make() below, in the event that we do a build.
result = command.CommandResult()
result.return_code = 0
- if self.builder.in_tree:
+ if work_in_output or self.builder.in_tree:
out_dir = work_dir
else:
if self.per_board_out_dir:
args.extend(self.builder.toolchains.GetMakeArguments(brd))
args.extend(self.toolchain.MakeArgs())
+ # Remove any output targets. Since we use a build directory that
+ # was previously used by another board, it may have produced an
+ # SPL image. If we don't remove it (i.e. see do_config and
+ # self.mrproper below) then it will appear to be the output of
+ # this build, even if it does not produce SPL images.
+ build_dir = self.builder.GetBuildDir(commit_upto, brd.target)
+ for elf in BASE_ELF_FILENAMES:
+ fname = os.path.join(out_dir, elf)
+ if os.path.exists(fname):
+ os.remove(fname)
+
# If we need to reconfigure, do that now
- if do_config:
+ cfg_file = os.path.join(out_dir, '.config')
+ if do_config or adjust_cfg:
config_out = ''
- if not self.incremental:
+ if self.mrproper:
result = self.Make(commit, brd, 'mrproper', cwd,
'mrproper', *args, env=env)
config_out += result.combined
*(args + config_args), env=env)
config_out += result.combined
do_config = False # No need to configure next time
+ if adjust_cfg:
+ cfgutil.adjust_cfg_file(cfg_file, adjust_cfg)
if result.return_code == 0:
if config_only:
args.append('cfg')
result = self.Make(commit, brd, 'build', cwd, *args,
env=env)
+ if adjust_cfg:
+ errs = cfgutil.check_cfg_file(cfg_file, adjust_cfg)
+ if errs:
+ print('errs', errs)
+ result.stderr += errs
+ result.return_code = 1
result.stderr = result.stderr.replace(src_dir + '/', '')
if self.builder.verbose_build:
result.stdout = config_out + result.stdout
result.out_dir = out_dir
return result, do_config
- def _WriteResult(self, result, keep_outputs):
+ def _WriteResult(self, result, keep_outputs, work_in_output):
"""Write a built result to the output directory.
Args:
result: CommandResult object containing result to write
keep_outputs: True to store the output binaries, False
to delete them
+ work_in_output: Use the output directory as the work directory and
+ don't write to a separate output directory.
"""
- # Fatal error
- if result.return_code < 0:
- return
-
# If we think this might have been aborted with Ctrl-C, record the
# failure but not that we are 'done' with this board. A retry may fix
# it.
- maybe_aborted = result.stderr and 'No child processes' in result.stderr
+ maybe_aborted = result.stderr and 'No child processes' in result.stderr
- if result.already_done:
+ if result.return_code >= 0 and result.already_done:
return
# Write the output and stderr
outfile = os.path.join(build_dir, 'log')
with open(outfile, 'w') as fd:
if result.stdout:
- # We don't want unicode characters in log files
- fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
+ fd.write(result.stdout)
errfile = self.builder.GetErrFile(result.commit_upto,
result.brd.target)
if result.stderr:
with open(errfile, 'w') as fd:
- # We don't want unicode characters in log files
- fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
+ fd.write(result.stderr)
elif os.path.exists(errfile):
os.remove(errfile)
+ # Fatal error
+ if result.return_code < 0:
+ return
+
if result.toolchain:
# Write the build result and toolchain information.
done_file = self.builder.GetDoneFile(result.commit_upto,
else:
fd.write('%s' % result.return_code)
with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
- print >>fd, 'gcc', result.toolchain.gcc
- print >>fd, 'path', result.toolchain.path
- print >>fd, 'cross', result.toolchain.cross
- print >>fd, 'arch', result.toolchain.arch
+ print('gcc', result.toolchain.gcc, file=fd)
+ print('path', result.toolchain.path, file=fd)
+ print('cross', result.toolchain.cross, file=fd)
+ print('arch', result.toolchain.arch, file=fd)
fd.write('%s' % result.return_code)
# Write out the image and function size information and an objdump
env = result.toolchain.MakeEnvironment(self.builder.full_path)
- with open(os.path.join(build_dir, 'env'), 'w') as fd:
+ with open(os.path.join(build_dir, 'out-env'), 'wb') as fd:
for var in sorted(env.keys()):
- print >>fd, '%s="%s"' % (var, env[var])
+ fd.write(b'%s="%s"' % (var, env[var]))
lines = []
- for fname in ['u-boot', 'spl/u-boot-spl']:
+ for fname in BASE_ELF_FILENAMES:
cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
nm_result = command.RunPipe([cmd], capture=True,
capture_stderr=True, cwd=result.out_dir,
nm = self.builder.GetFuncSizesFile(result.commit_upto,
result.brd.target, fname)
with open(nm, 'w') as fd:
- print >>fd, nm_result.stdout,
+ print(nm_result.stdout, end=' ', file=fd)
cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
dump_result = command.RunPipe([cmd], capture=True,
objdump = self.builder.GetObjdumpFile(result.commit_upto,
result.brd.target, fname)
with open(objdump, 'w') as fd:
- print >>fd, dump_result.stdout,
+ print(dump_result.stdout, end=' ', file=fd)
for line in dump_result.stdout.splitlines():
fields = line.split()
if len(fields) > 5 and fields[1] == '.rodata':
capture_stderr=True, cwd=result.out_dir,
raise_on_error=False, env=env)
ubootenv = os.path.join(result.out_dir, 'uboot.env')
- self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
+ if not work_in_output:
+ self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
# Write out the image sizes file. This is similar to the output
# of binutil's 'size' utility, but it omits the header line and
sizes = self.builder.GetSizesFile(result.commit_upto,
result.brd.target)
with open(sizes, 'w') as fd:
- print >>fd, '\n'.join(lines)
-
- # Write out the configuration files, with a special case for SPL
- for dirname in ['', 'spl', 'tpl']:
- self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
- 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
- 'include/autoconf.mk', 'include/generated/autoconf.h'])
-
- # Now write the actual build output
- if keep_outputs:
- self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
- '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
- 'spl/u-boot-spl*'])
+ print('\n'.join(lines), file=fd)
+
+ if not work_in_output:
+ # Write out the configuration files, with a special case for SPL
+ for dirname in ['', 'spl', 'tpl']:
+ self.CopyFiles(
+ result.out_dir, build_dir, dirname,
+ ['u-boot.cfg', 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg',
+ '.config', 'include/autoconf.mk',
+ 'include/generated/autoconf.h'])
+
+ # Now write the actual build output
+ if keep_outputs:
+ self.CopyFiles(
+ result.out_dir, build_dir, '',
+ ['u-boot*', '*.bin', '*.map', '*.img', 'MLO', 'SPL',
+ 'include/autoconf.mk', 'spl/u-boot-spl*'])
def CopyFiles(self, out_dir, build_dir, dirname, patterns):
"""Copy files from the build directory to the output.
target = '%s-%s%s' % (base, dirname, ext)
shutil.copy(fname, os.path.join(build_dir, target))
+ def _SendResult(self, result):
+ """Send a result to the builder for processing
+
+ Args:
+ result: CommandResult object containing the results of the build
+
+ Raises:
+ ValueError if self.test_exception is true (for testing)
+ """
+ if self.test_exception:
+ raise ValueError('test exception')
+ if self.thread_num != -1:
+ self.builder.out_queue.put(result)
+ else:
+ self.builder.ProcessResult(result)
+
def RunJob(self, job):
"""Run a single job
Args:
job: Job to build
+
+ Returns:
+ List of Result objects
"""
brd = job.board
work_dir = self.builder.GetThreadDir(self.thread_num)
result, request_config = self.RunCommit(commit_upto, brd,
work_dir, do_config, self.builder.config_only,
force_build or self.builder.force_build,
- self.builder.force_build_failures)
+ self.builder.force_build_failures,
+ job.work_in_output, job.adjust_cfg)
failed = result.return_code or result.stderr
did_config = do_config
if failed and not do_config:
# with a reconfig.
if self.builder.force_config_on_failure:
result, request_config = self.RunCommit(commit_upto,
- brd, work_dir, True, False, True, False)
+ brd, work_dir, True, False, True, False,
+ job.work_in_output, job.adjust_cfg)
did_config = True
if not self.builder.force_reconfig:
do_config = request_config
raise ValueError('Interrupt')
# We have the build results, so output the result
- self._WriteResult(result, job.keep_outputs)
- self.builder.out_queue.put(result)
+ self._WriteResult(result, job.keep_outputs, job.work_in_output)
+ self._SendResult(result)
else:
# Just build the currently checked-out build
result, request_config = self.RunCommit(None, brd, work_dir, True,
self.builder.config_only, True,
- self.builder.force_build_failures)
+ self.builder.force_build_failures, job.work_in_output,
+ job.adjust_cfg)
result.commit_upto = 0
- self._WriteResult(result, job.keep_outputs)
- self.builder.out_queue.put(result)
+ self._WriteResult(result, job.keep_outputs, job.work_in_output)
+ self._SendResult(result)
def run(self):
"""Our thread's run function
"""
while True:
job = self.builder.queue.get()
- self.RunJob(job)
+ try:
+ self.RunJob(job)
+ except Exception as e:
+ print('Thread exception (use -T0 to run without threads):', e)
+ self.builder.thread_exceptions.append(e)
self.builder.queue.task_done()
projects / J-u-boot.git / blobdiff