[J-u-boot.git] / tools / buildman / builderthread.py

# Copyright (c) 2014 Google, Inc
#
# SPDX-License-Identifier:      GPL-2.0+
#

import errno
import glob
import os
import shutil
import threading

import command
import gitutil

def Mkdir(dirname):
    """Make a directory if it doesn't already exist.

    Args:
        dirname: Directory to create
    """
    try:
        os.mkdir(dirname)
    except OSError as err:
        if err.errno == errno.EEXIST:
            pass
        else:
            raise

class BuilderJob:
    """Holds information about a job to be performed by a thread

    Members:
        board: Board object to build
        commits: List of commit options to build.
    """
    def __init__(self):
        self.board = None
        self.commits = []


class ResultThread(threading.Thread):
    """This thread processes results from builder threads.

    It simply passes the results on to the builder. There is only one
    result thread, and this helps to serialise the build output.
    """
    def __init__(self, builder):
        """Set up a new result thread

        Args:
            builder: Builder which will be sent each result
        """
        threading.Thread.__init__(self)
        self.builder = builder

    def run(self):
        """Called to start up the result thread.

        We collect the next result job and pass it on to the build.
        """
        while True:
            result = self.builder.out_queue.get()
            self.builder.ProcessResult(result)
            self.builder.out_queue.task_done()


class BuilderThread(threading.Thread):
    """This thread builds U-Boot for a particular board.

    An input queue provides each new job. We run 'make' to build U-Boot
    and then pass the results on to the output queue.

    Members:
        builder: The builder which contains information we might need
        thread_num: Our thread number (0-n-1), used to decide on a
                temporary directory
    """
    def __init__(self, builder, thread_num):
        """Set up a new builder thread"""
        threading.Thread.__init__(self)
        self.builder = builder
        self.thread_num = thread_num

    def Make(self, commit, brd, stage, cwd, *args, **kwargs):
        """Run 'make' on a particular commit and board.

        The source code will already be checked out, so the 'commit'
        argument is only for information.

        Args:
            commit: Commit object that is being built
            brd: Board object that is being built
            stage: Stage of the build. Valid stages are:
                        mrproper - can be called to clean source
                        config - called to configure for a board
                        build - the main make invocation - it does the build
            args: A list of arguments to pass to 'make'
            kwargs: A list of keyword arguments to pass to command.RunPipe()

        Returns:
            CommandResult object
        """
        return self.builder.do_make(commit, brd, stage, cwd, *args,
                **kwargs)

    def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
                  force_build_failures):
        """Build a particular commit.

        If the build is already done, and we are not forcing a build, we skip
        the build and just return the previously-saved results.

        Args:
            commit_upto: Commit number to build (0...n-1)
            brd: Board object to build
            work_dir: Directory to which the source will be checked out
            do_config: True to run a make <board>_defconfig on the source
            force_build: Force a build even if one was previously done
            force_build_failures: Force a bulid if the previous result showed
                failure

        Returns:
            tuple containing:
                - CommandResult object containing the results of the build
                - boolean indicating whether 'make config' is still needed
        """
        # Create a default result - it will be overwritte by the call to
        # self.Make() below, in the event that we do a build.
        result = command.CommandResult()
        result.return_code = 0
        if self.builder.in_tree:
            out_dir = work_dir
        else:
            out_dir = os.path.join(work_dir, 'build')

        # Check if the job was already completed last time
        done_file = self.builder.GetDoneFile(commit_upto, brd.target)
        result.already_done = os.path.exists(done_file)
        will_build = (force_build or force_build_failures or
            not result.already_done)
        if result.already_done and will_build:
            # Get the return code from that build and use it
            with open(done_file, 'r') as fd:
                result.return_code = int(fd.readline())
            err_file = self.builder.GetErrFile(commit_upto, brd.target)
            if os.path.exists(err_file) and os.stat(err_file).st_size:
                result.stderr = 'bad'
            elif not force_build:
                # The build passed, so no need to build it again
                will_build = False

        if will_build:
            # We are going to have to build it. First, get a toolchain
            if not self.toolchain:
                try:
                    self.toolchain = self.builder.toolchains.Select(brd.arch)
                except ValueError as err:
                    result.return_code = 10
                    result.stdout = ''
                    result.stderr = str(err)
                    # TODO([email protected]): This gets swallowed, but needs
                    # to be reported.

            if self.toolchain:
                # Checkout the right commit
                if self.builder.commits:
                    commit = self.builder.commits[commit_upto]
                    if self.builder.checkout:
                        git_dir = os.path.join(work_dir, '.git')
                        gitutil.Checkout(commit.hash, git_dir, work_dir,
                                         force=True)
                else:
                    commit = 'current'

                # Set up the environment and command line
                env = self.toolchain.MakeEnvironment()
                Mkdir(out_dir)
                args = []
                cwd = work_dir
                if not self.builder.in_tree:
                    if commit_upto is None:
                        # In this case we are building in the original source
                        # directory (i.e. the current directory where buildman
                        # is invoked. The output directory is set to this
                        # thread's selected work directory.
                        #
                        # Symlinks can confuse U-Boot's Makefile since
                        # we may use '..' in our path, so remove them.
                        work_dir = os.path.realpath(work_dir)
                        args.append('O=%s/build' % work_dir)
                        cwd = None
                    else:
                        args.append('O=build')
                args.append('-s')
                if self.builder.num_jobs is not None:
                    args.extend(['-j', str(self.builder.num_jobs)])
                config_args = ['%s_defconfig' % brd.target]
                config_out = ''
                args.extend(self.builder.toolchains.GetMakeArguments(brd))

                # If we need to reconfigure, do that now
                if do_config:
                    result = self.Make(commit, brd, 'mrproper', cwd,
                            'mrproper', *args, env=env)
                    result = self.Make(commit, brd, 'config', cwd,
                            *(args + config_args), env=env)
                    config_out = result.combined
                    do_config = False   # No need to configure next time
                if result.return_code == 0:
                    result = self.Make(commit, brd, 'build', cwd, *args,
                            env=env)
                    result.stdout = config_out + result.stdout
            else:
                result.return_code = 1
                result.stderr = 'No tool chain for %s\n' % brd.arch
            result.already_done = False

        result.toolchain = self.toolchain
        result.brd = brd
        result.commit_upto = commit_upto
        result.out_dir = out_dir
        return result, do_config

    def _WriteResult(self, result, keep_outputs):
        """Write a built result to the output directory.

        Args:
            result: CommandResult object containing result to write
            keep_outputs: True to store the output binaries, False
                to delete them
        """
        # Fatal error
        if result.return_code < 0:
            return

        # Aborted?
        if result.stderr and 'No child processes' in result.stderr:
            return

        if result.already_done:
            return

        # Write the output and stderr
        output_dir = self.builder._GetOutputDir(result.commit_upto)
        Mkdir(output_dir)
        build_dir = self.builder.GetBuildDir(result.commit_upto,
                result.brd.target)
        Mkdir(build_dir)

        outfile = os.path.join(build_dir, 'log')
        with open(outfile, 'w') as fd:
            if result.stdout:
                fd.write(result.stdout)

        errfile = self.builder.GetErrFile(result.commit_upto,
                result.brd.target)
        if result.stderr:
            with open(errfile, 'w') as fd:
                fd.write(result.stderr)
        elif os.path.exists(errfile):
            os.remove(errfile)

        if result.toolchain:
            # Write the build result and toolchain information.
            done_file = self.builder.GetDoneFile(result.commit_upto,
                    result.brd.target)
            with open(done_file, 'w') as fd:
                fd.write('%s' % result.return_code)
            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
                print >>fd, 'gcc', result.toolchain.gcc
                print >>fd, 'path', result.toolchain.path
                print >>fd, 'cross', result.toolchain.cross
                print >>fd, 'arch', result.toolchain.arch
                fd.write('%s' % result.return_code)

            with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
                print >>fd, 'gcc', result.toolchain.gcc
                print >>fd, 'path', result.toolchain.path

            # Write out the image and function size information and an objdump
            env = result.toolchain.MakeEnvironment()
            lines = []
            for fname in ['u-boot', 'spl/u-boot-spl']:
                cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
                nm_result = command.RunPipe([cmd], capture=True,
                        capture_stderr=True, cwd=result.out_dir,
                        raise_on_error=False, env=env)
                if nm_result.stdout:
                    nm = self.builder.GetFuncSizesFile(result.commit_upto,
                                    result.brd.target, fname)
                    with open(nm, 'w') as fd:
                        print >>fd, nm_result.stdout,

                cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
                dump_result = command.RunPipe([cmd], capture=True,
                        capture_stderr=True, cwd=result.out_dir,
                        raise_on_error=False, env=env)
                rodata_size = ''
                if dump_result.stdout:
                    objdump = self.builder.GetObjdumpFile(result.commit_upto,
                                    result.brd.target, fname)
                    with open(objdump, 'w') as fd:
                        print >>fd, dump_result.stdout,
                    for line in dump_result.stdout.splitlines():
                        fields = line.split()
                        if len(fields) > 5 and fields[1] == '.rodata':
                            rodata_size = fields[2]

                cmd = ['%ssize' % self.toolchain.cross, fname]
                size_result = command.RunPipe([cmd], capture=True,
                        capture_stderr=True, cwd=result.out_dir,
                        raise_on_error=False, env=env)
                if size_result.stdout:
                    lines.append(size_result.stdout.splitlines()[1] + ' ' +
                                 rodata_size)

            # Write out the image sizes file. This is similar to the output
            # of binutil's 'size' utility, but it omits the header line and
            # adds an additional hex value at the end of each line for the
            # rodata size
            if len(lines):
                sizes = self.builder.GetSizesFile(result.commit_upto,
                                result.brd.target)
                with open(sizes, 'w') as fd:
                    print >>fd, '\n'.join(lines)

        # Now write the actual build output
        if keep_outputs:
            patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
                        'include/autoconf.mk', 'spl/u-boot-spl',
                        'spl/u-boot-spl.bin']
            for pattern in patterns:
                file_list = glob.glob(os.path.join(result.out_dir, pattern))
                for fname in file_list:
                    shutil.copy(fname, build_dir)


    def RunJob(self, job):
        """Run a single job

        A job consists of a building a list of commits for a particular board.

        Args:
            job: Job to build
        """
        brd = job.board
        work_dir = self.builder.GetThreadDir(self.thread_num)
        self.toolchain = None
        if job.commits:
            # Run 'make board_defconfig' on the first commit
            do_config = True
            commit_upto  = 0
            force_build = False
            for commit_upto in range(0, len(job.commits), job.step):
                result, request_config = self.RunCommit(commit_upto, brd,
                        work_dir, do_config,
                        force_build or self.builder.force_build,
                        self.builder.force_build_failures)
                failed = result.return_code or result.stderr
                did_config = do_config
                if failed and not do_config:
                    # If our incremental build failed, try building again
                    # with a reconfig.
                    if self.builder.force_config_on_failure:
                        result, request_config = self.RunCommit(commit_upto,
                            brd, work_dir, True, True, False)
                        did_config = True
                if not self.builder.force_reconfig:
                    do_config = request_config

                # If we built that commit, then config is done. But if we got
                # an warning, reconfig next time to force it to build the same
                # files that created warnings this time. Otherwise an
                # incremental build may not build the same file, and we will
                # think that the warning has gone away.
                # We could avoid this by using -Werror everywhere...
                # For errors, the problem doesn't happen, since presumably
                # the build stopped and didn't generate output, so will retry
                # that file next time. So we could detect warnings and deal
                # with them specially here. For now, we just reconfigure if
                # anything goes work.
                # Of course this is substantially slower if there are build
                # errors/warnings (e.g. 2-3x slower even if only 10% of builds
                # have problems).
                if (failed and not result.already_done and not did_config and
                        self.builder.force_config_on_failure):
                    # If this build failed, try the next one with a
                    # reconfigure.
                    # Sometimes if the board_config.h file changes it can mess
                    # with dependencies, and we get:
                    # make: *** No rule to make target `include/autoconf.mk',
                    #     needed by `depend'.
                    do_config = True
                    force_build = True
                else:
                    force_build = False
                    if self.builder.force_config_on_failure:
                        if failed:
                            do_config = True
                    result.commit_upto = commit_upto
                    if result.return_code < 0:
                        raise ValueError('Interrupt')

                # We have the build results, so output the result
                self._WriteResult(result, job.keep_outputs)
                self.builder.out_queue.put(result)
        else:
            # Just build the currently checked-out build
            result, request_config = self.RunCommit(None, brd, work_dir, True,
                        True, self.builder.force_build_failures)
            result.commit_upto = 0
            self._WriteResult(result, job.keep_outputs)
            self.builder.out_queue.put(result)

    def run(self):
        """Our thread's run function

        This thread picks a job from the queue, runs it, and then goes to the
        next job.
        """
        alive = True
        while True:
            job = self.builder.queue.get()
            if self.builder.active and alive:
                self.RunJob(job)
            '''
            try:
                if self.builder.active and alive:
                    self.RunJob(job)
            except Exception as err:
                alive = False
                print err
            '''
            self.builder.queue.task_done()
Commit	Line	Data
190064b4 SG	1	# Copyright (c) 2014 Google, Inc
	2	#
	3	# SPDX-License-Identifier: GPL-2.0+
	4	#
	5
	6	import errno
	7	import glob
	8	import os
	9	import shutil
	10	import threading
	11
	12	import command
	13	import gitutil
	14
	15	def Mkdir(dirname):
	16	"""Make a directory if it doesn't already exist.
	17
	18	Args:
	19	dirname: Directory to create
	20	"""
	21	try:
	22	os.mkdir(dirname)
	23	except OSError as err:
	24	if err.errno == errno.EEXIST:
	25	pass
	26	else:
	27	raise
	28
	29	class BuilderJob:
	30	"""Holds information about a job to be performed by a thread
	31
	32	Members:
	33	board: Board object to build
	34	commits: List of commit options to build.
	35	"""
	36	def __init__(self):
	37	self.board = None
	38	self.commits = []
	39
	40
	41	class ResultThread(threading.Thread):
	42	"""This thread processes results from builder threads.
	43
	44	It simply passes the results on to the builder. There is only one
	45	result thread, and this helps to serialise the build output.
	46	"""
	47	def __init__(self, builder):
	48	"""Set up a new result thread
	49
	50	Args:
	51	builder: Builder which will be sent each result
	52	"""
	53	threading.Thread.__init__(self)
	54	self.builder = builder
	55
	56	def run(self):
	57	"""Called to start up the result thread.
	58
	59	We collect the next result job and pass it on to the build.
	60	"""
	61	while True:
	62	result = self.builder.out_queue.get()
	63	self.builder.ProcessResult(result)
	64	self.builder.out_queue.task_done()
65
66
67	class BuilderThread(threading.Thread):
68	"""This thread builds U-Boot for a particular board.
69
70	An input queue provides each new job. We run 'make' to build U-Boot
71	and then pass the results on to the output queue.
72
73	Members:
74	builder: The builder which contains information we might need
75	thread_num: Our thread number (0-n-1), used to decide on a
76	temporary directory
77	"""
78	def __init__(self, builder, thread_num):
79	"""Set up a new builder thread"""
80	threading.Thread.__init__(self)
81	self.builder = builder
82	self.thread_num = thread_num
83
84	def Make(self, commit, brd, stage, cwd, args, *kwargs):
85	"""Run 'make' on a particular commit and board.
86
87	The source code will already be checked out, so the 'commit'
88	argument is only for information.
89
90	Args:
91	commit: Commit object that is being built
92	brd: Board object that is being built
93	stage: Stage of the build. Valid stages are:
fd18a89e	94	mrproper - can be called to clean source
190064b4 SG	95	config - called to configure for a board
	96	build - the main make invocation - it does the build
	97	args: A list of arguments to pass to 'make'
	98	kwargs: A list of keyword arguments to pass to command.RunPipe()
	99
	100	Returns:
	101	CommandResult object
	102	"""
	103	return self.builder.do_make(commit, brd, stage, cwd, *args,
	104	**kwargs)
	105
	106	def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
	107	force_build_failures):
	108	"""Build a particular commit.
	109
	110	If the build is already done, and we are not forcing a build, we skip
	111	the build and just return the previously-saved results.
	112
	113	Args:
	114	commit_upto: Commit number to build (0...n-1)
	115	brd: Board object to build
	116	work_dir: Directory to which the source will be checked out
	117	do_config: True to run a make <board>_defconfig on the source
	118	force_build: Force a build even if one was previously done
	119	force_build_failures: Force a bulid if the previous result showed
	120	failure
	121
	122	Returns:
	123	tuple containing:
	124	- CommandResult object containing the results of the build
	125	- boolean indicating whether 'make config' is still needed
	126	"""
	127	# Create a default result - it will be overwritte by the call to
	128	# self.Make() below, in the event that we do a build.
	129	result = command.CommandResult()
	130	result.return_code = 0
	131	if self.builder.in_tree:
	132	out_dir = work_dir
	133	else:
	134	out_dir = os.path.join(work_dir, 'build')
	135
	136	# Check if the job was already completed last time
	137	done_file = self.builder.GetDoneFile(commit_upto, brd.target)
	138	result.already_done = os.path.exists(done_file)
	139	will_build = (force_build or force_build_failures or
	140	not result.already_done)
	141	if result.already_done and will_build:
	142	# Get the return code from that build and use it
	143	with open(done_file, 'r') as fd:
	144	result.return_code = int(fd.readline())
	145	err_file = self.builder.GetErrFile(commit_upto, brd.target)
	146	if os.path.exists(err_file) and os.stat(err_file).st_size:
	147	result.stderr = 'bad'
	148	elif not force_build:
	149	# The build passed, so no need to build it again
	150	will_build = False
	151
	152	if will_build:
	153	# We are going to have to build it. First, get a toolchain
	154	if not self.toolchain:
	155	try:
	156	self.toolchain = self.builder.toolchains.Select(brd.arch)
	157	except ValueError as err:
	158	result.return_code = 10
159	result.stdout = ''
160	result.stderr = str(err)
161	# TODO([email protected]): This gets swallowed, but needs
162	# to be reported.
163
164	if self.toolchain:
165	# Checkout the right commit
166	if self.builder.commits:
167	commit = self.builder.commits[commit_upto]
168	if self.builder.checkout:
169	git_dir = os.path.join(work_dir, '.git')
170	gitutil.Checkout(commit.hash, git_dir, work_dir,
171	force=True)
172	else:
173	commit = 'current'
174
175	# Set up the environment and command line
176	env = self.toolchain.MakeEnvironment()
177	Mkdir(out_dir)
178	args = []
179	cwd = work_dir
180	if not self.builder.in_tree:
181	if commit_upto is None:
182	# In this case we are building in the original source
183	# directory (i.e. the current directory where buildman
184	# is invoked. The output directory is set to this
185	# thread's selected work directory.
186	#
187	# Symlinks can confuse U-Boot's Makefile since
188	# we may use '..' in our path, so remove them.
189	work_dir = os.path.realpath(work_dir)
190	args.append('O=%s/build' % work_dir)
191	cwd = None
192	else:
193	args.append('O=build')
194	args.append('-s')
195	if self.builder.num_jobs is not None:
196	args.extend(['-j', str(self.builder.num_jobs)])
197	config_args = ['%s_defconfig' % brd.target]
198	config_out = ''
199	args.extend(self.builder.toolchains.GetMakeArguments(brd))
200
201	# If we need to reconfigure, do that now
202	if do_config:
fd18a89e RM	203	result = self.Make(commit, brd, 'mrproper', cwd,
fd18a89e RM	204	'mrproper', *args, env=env)
190064b4 SG	205	result = self.Make(commit, brd, 'config', cwd,
	206	*(args + config_args), env=env)
	207	config_out = result.combined
	208	do_config = False # No need to configure next time
	209	if result.return_code == 0:
	210	result = self.Make(commit, brd, 'build', cwd, *args,
	211	env=env)
	212	result.stdout = config_out + result.stdout
	213	else:
	214	result.return_code = 1
	215	result.stderr = 'No tool chain for %s\n' % brd.arch
	216	result.already_done = False
	217
	218	result.toolchain = self.toolchain
	219	result.brd = brd
	220	result.commit_upto = commit_upto
	221	result.out_dir = out_dir
	222	return result, do_config
	223
	224	def _WriteResult(self, result, keep_outputs):
	225	"""Write a built result to the output directory.
	226
	227	Args:
	228	result: CommandResult object containing result to write
	229	keep_outputs: True to store the output binaries, False
	230	to delete them
	231	"""
	232	# Fatal error
	233	if result.return_code < 0:
	234	return
	235
	236	# Aborted?
	237	if result.stderr and 'No child processes' in result.stderr:
	238	return
	239
	240	if result.already_done:
	241	return
	242
	243	# Write the output and stderr
	244	output_dir = self.builder._GetOutputDir(result.commit_upto)
	245	Mkdir(output_dir)
	246	build_dir = self.builder.GetBuildDir(result.commit_upto,
	247	result.brd.target)
	248	Mkdir(build_dir)
	249
	250	outfile = os.path.join(build_dir, 'log')
	251	with open(outfile, 'w') as fd:
	252	if result.stdout:
	253	fd.write(result.stdout)
	254
	255	errfile = self.builder.GetErrFile(result.commit_upto,
	256	result.brd.target)
	257	if result.stderr:
	258	with open(errfile, 'w') as fd:
	259	fd.write(result.stderr)
	260	elif os.path.exists(errfile):
	261	os.remove(errfile)
	262
	263	if result.toolchain:
	264	# Write the build result and toolchain information.
	265	done_file = self.builder.GetDoneFile(result.commit_upto,
	266	result.brd.target)
	267	with open(done_file, 'w') as fd:
	268	fd.write('%s' % result.return_code)
269	with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
270	print >>fd, 'gcc', result.toolchain.gcc
271	print >>fd, 'path', result.toolchain.path
272	print >>fd, 'cross', result.toolchain.cross
273	print >>fd, 'arch', result.toolchain.arch
274	fd.write('%s' % result.return_code)
275
276	with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
277	print >>fd, 'gcc', result.toolchain.gcc
278	print >>fd, 'path', result.toolchain.path
279
280	# Write out the image and function size information and an objdump
281	env = result.toolchain.MakeEnvironment()
282	lines = []
283	for fname in ['u-boot', 'spl/u-boot-spl']:
284	cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
285	nm_result = command.RunPipe([cmd], capture=True,
286	capture_stderr=True, cwd=result.out_dir,
287	raise_on_error=False, env=env)
288	if nm_result.stdout:
289	nm = self.builder.GetFuncSizesFile(result.commit_upto,
290	result.brd.target, fname)
291	with open(nm, 'w') as fd:
292	print >>fd, nm_result.stdout,
293
294	cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
295	dump_result = command.RunPipe([cmd], capture=True,
296	capture_stderr=True, cwd=result.out_dir,
297	raise_on_error=False, env=env)
298	rodata_size = ''
299	if dump_result.stdout:
300	objdump = self.builder.GetObjdumpFile(result.commit_upto,
301	result.brd.target, fname)
302	with open(objdump, 'w') as fd:
303	print >>fd, dump_result.stdout,
304	for line in dump_result.stdout.splitlines():
305	fields = line.split()
306	if len(fields) > 5 and fields[1] == '.rodata':
307	rodata_size = fields[2]
308
309	cmd = ['%ssize' % self.toolchain.cross, fname]
310	size_result = command.RunPipe([cmd], capture=True,
311	capture_stderr=True, cwd=result.out_dir,
312	raise_on_error=False, env=env)
313	if size_result.stdout:
314	lines.append(size_result.stdout.splitlines()[1] + ' ' +
315	rodata_size)
316
317	# Write out the image sizes file. This is similar to the output
318	# of binutil's 'size' utility, but it omits the header line and
319	# adds an additional hex value at the end of each line for the
320	# rodata size
321	if len(lines):
322	sizes = self.builder.GetSizesFile(result.commit_upto,
323	result.brd.target)
324	with open(sizes, 'w') as fd:
325	print >>fd, '\n'.join(lines)
326
327	# Now write the actual build output
328	if keep_outputs:
329	patterns = ['u-boot', '.bin', 'u-boot.dtb', '.map',
330	'include/autoconf.mk', 'spl/u-boot-spl',
331	'spl/u-boot-spl.bin']
332	for pattern in patterns:
333	file_list = glob.glob(os.path.join(result.out_dir, pattern))
334	for fname in file_list:
335	shutil.copy(fname, build_dir)
336
337
338	def RunJob(self, job):
339	"""Run a single job
340
341	A job consists of a building a list of commits for a particular board.
342
343	Args:
344	job: Job to build
345	"""
346	brd = job.board
347	work_dir = self.builder.GetThreadDir(self.thread_num)
348	self.toolchain = None
349	if job.commits:
350	# Run 'make board_defconfig' on the first commit
351	do_config = True
352	commit_upto = 0
353	force_build = False
354	for commit_upto in range(0, len(job.commits), job.step):
355	result, request_config = self.RunCommit(commit_upto, brd,
356	work_dir, do_config,
357	force_build or self.builder.force_build,
358	self.builder.force_build_failures)
359	failed = result.return_code or result.stderr
360	did_config = do_config
361	if failed and not do_config:
362	# If our incremental build failed, try building again
363	# with a reconfig.
364	if self.builder.force_config_on_failure:
365	result, request_config = self.RunCommit(commit_upto,
366	brd, work_dir, True, True, False)
367	did_config = True
368	if not self.builder.force_reconfig:
369	do_config = request_config
370
371	# If we built that commit, then config is done. But if we got
372	# an warning, reconfig next time to force it to build the same
373	# files that created warnings this time. Otherwise an
374	# incremental build may not build the same file, and we will
375	# think that the warning has gone away.
376	# We could avoid this by using -Werror everywhere...
377	# For errors, the problem doesn't happen, since presumably
378	# the build stopped and didn't generate output, so will retry
379	# that file next time. So we could detect warnings and deal
380	# with them specially here. For now, we just reconfigure if
381	# anything goes work.
382	# Of course this is substantially slower if there are build
383	# errors/warnings (e.g. 2-3x slower even if only 10% of builds
384	# have problems).
385	if (failed and not result.already_done and not did_config and
386	self.builder.force_config_on_failure):
387	# If this build failed, try the next one with a
388	# reconfigure.
389	# Sometimes if the board_config.h file changes it can mess
390	# with dependencies, and we get:
391	# make: *** No rule to make target `include/autoconf.mk',
392	# needed by `depend'.
393	do_config = True
394	force_build = True
395	else:
396	force_build = False
397	if self.builder.force_config_on_failure:
398	if failed:
399	do_config = True
400	result.commit_upto = commit_upto
401	if result.return_code < 0:
402	raise ValueError('Interrupt')
403
404	# We have the build results, so output the result
405	self._WriteResult(result, job.keep_outputs)
406	self.builder.out_queue.put(result)
407	else:
408	# Just build the currently checked-out build
409	result, request_config = self.RunCommit(None, brd, work_dir, True,
410	True, self.builder.force_build_failures)
411	result.commit_upto = 0
412	self._WriteResult(result, job.keep_outputs)
413	self.builder.out_queue.put(result)
414
415	def run(self):
416	"""Our thread's run function
417
418	This thread picks a job from the queue, runs it, and then goes to the
419	next job.
420	"""
421	alive = True
422	while True:
423	job = self.builder.queue.get()
424	if self.builder.active and alive:
425	self.RunJob(job)
426	'''
427	try:
428	if self.builder.active and alive:
429	self.RunJob(job)
430	except Exception as err:
431	alive = False
432	print err
433	'''
434	self.builder.queue.task_done()