aboutsummaryrefslogtreecommitdiffstats
path: root/Task.py
diff options
context:
space:
mode:
Diffstat (limited to 'Task.py')
-rw-r--r--Task.py1394
1 files changed, 1394 insertions, 0 deletions
diff --git a/Task.py b/Task.py
new file mode 100644
index 0000000..0fc449d
--- /dev/null
+++ b/Task.py
@@ -0,0 +1,1394 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Thomas Nagy, 2005-2018 (ita)
+
+"""
+Tasks represent atomic operations such as processes.
+"""
+
+import os, re, sys, tempfile, traceback
+from waflib import Utils, Logs, Errors
+
+# task states
+NOT_RUN = 0
+"""The task was not executed yet"""
+
+MISSING = 1
+"""The task has been executed but the files have not been created"""
+
+CRASHED = 2
+"""The task execution returned a non-zero exit status"""
+
+EXCEPTION = 3
+"""An exception occurred in the task execution"""
+
+CANCELED = 4
+"""A dependency for the task is missing so it was cancelled"""
+
+SKIPPED = 8
+"""The task did not have to be executed"""
+
+SUCCESS = 9
+"""The task was successfully executed"""
+
+ASK_LATER = -1
+"""The task is not ready to be executed"""
+
+SKIP_ME = -2
+"""The task does not need to be executed"""
+
+RUN_ME = -3
+"""The task must be executed"""
+
+CANCEL_ME = -4
+"""The task cannot be executed because of a dependency problem"""
+
+COMPILE_TEMPLATE_SHELL = '''
+def f(tsk):
+ env = tsk.env
+ gen = tsk.generator
+ bld = gen.bld
+ cwdx = tsk.get_cwd()
+ p = env.get_flat
+ tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
+ return tsk.exec_command(cmd, cwd=cwdx, env=env.env or None)
+'''
+
+COMPILE_TEMPLATE_NOSHELL = '''
+def f(tsk):
+ env = tsk.env
+ gen = tsk.generator
+ bld = gen.bld
+ cwdx = tsk.get_cwd()
+ def to_list(xx):
+ if isinstance(xx, str): return [xx]
+ return xx
+ def merge(lst1, lst2):
+ if lst1 and lst2:
+ return lst1[:-1] + [lst1[-1] + lst2[0]] + lst2[1:]
+ return lst1 + lst2
+ lst = []
+ %s
+ if '' in lst:
+ lst = [x for x in lst if x]
+ tsk.last_cmd = lst
+ return tsk.exec_command(lst, cwd=cwdx, env=env.env or None)
+'''
+
+COMPILE_TEMPLATE_SIG_VARS = '''
+def f(tsk):
+ super(tsk.__class__, tsk).sig_vars()
+ env = tsk.env
+ gen = tsk.generator
+ bld = gen.bld
+ cwdx = tsk.get_cwd()
+ p = env.get_flat
+ buf = []
+ %s
+ tsk.m.update(repr(buf).encode())
+'''
+
+classes = {}
+"""
+The metaclass :py:class:`waflib.Task.store_task_type` stores all class tasks
+created by user scripts or Waf tools to this dict. It maps class names to class objects.
+"""
+
+class store_task_type(type):
+ """
+ Metaclass: store the task classes into the dict pointed by the
+ class attribute 'register' which defaults to :py:const:`waflib.Task.classes`,
+
+ The attribute 'run_str' is compiled into a method 'run' bound to the task class.
+ """
+ def __init__(cls, name, bases, dict):
+ super(store_task_type, cls).__init__(name, bases, dict)
+ name = cls.__name__
+
+ if name != 'evil' and name != 'Task':
+ if getattr(cls, 'run_str', None):
+ # if a string is provided, convert it to a method
+ (f, dvars) = compile_fun(cls.run_str, cls.shell)
+ cls.hcode = Utils.h_cmd(cls.run_str)
+ cls.orig_run_str = cls.run_str
+ # change the name of run_str or it is impossible to subclass with a function
+ cls.run_str = None
+ cls.run = f
+ # process variables
+ cls.vars = list(set(cls.vars + dvars))
+ cls.vars.sort()
+ if cls.vars:
+ fun = compile_sig_vars(cls.vars)
+ if fun:
+ cls.sig_vars = fun
+ elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__:
+ # getattr(cls, 'hcode') would look in the upper classes
+ cls.hcode = Utils.h_cmd(cls.run)
+
+ # be creative
+ getattr(cls, 'register', classes)[name] = cls
+
+evil = store_task_type('evil', (object,), {})
+"Base class provided to avoid writing a metaclass, so the code can run in python 2.6 and 3.x unmodified"
+
+class Task(evil):
+ """
+ Task objects represents actions to perform such as commands to execute by calling the `run` method.
+
+ Detecting when to execute a task occurs in the method :py:meth:`waflib.Task.Task.runnable_status`.
+
+ Detecting which tasks to execute is performed through a hash value returned by
+ :py:meth:`waflib.Task.Task.signature`. The task signature is persistent from build to build.
+ """
+ vars = []
+ """ConfigSet variables that should trigger a rebuild (class attribute used for :py:meth:`waflib.Task.Task.sig_vars`)"""
+
+ always_run = False
+ """Specify whether task instances must always be executed or not (class attribute)"""
+
+ shell = False
+ """Execute the command with the shell (class attribute)"""
+
+ color = 'GREEN'
+ """Color for the console display, see :py:const:`waflib.Logs.colors_lst`"""
+
+ ext_in = []
+ """File extensions that objects of this task class may use"""
+
+ ext_out = []
+ """File extensions that objects of this task class may create"""
+
+ before = []
+ """List of task class names to execute before instances of this class"""
+
+ after = []
+ """List of task class names to execute after instances of this class"""
+
+ hcode = Utils.SIG_NIL
+ """String representing an additional hash for the class representation"""
+
+ keep_last_cmd = False
+ """Whether to keep the last command executed on the instance after execution.
+ This may be useful for certain extensions but it can a lot of memory.
+ """
+
+ weight = 0
+ """Optional weight to tune the priority for task instances.
+ The higher, the earlier. The weight only applies to single task objects."""
+
+ tree_weight = 0
+ """Optional weight to tune the priority of task instances and whole subtrees.
+ The higher, the earlier."""
+
+ prio_order = 0
+ """Priority order set by the scheduler on instances during the build phase.
+ You most likely do not need to set it.
+ """
+
+ __slots__ = ('hasrun', 'generator', 'env', 'inputs', 'outputs', 'dep_nodes', 'run_after')
+
+ def __init__(self, *k, **kw):
+ self.hasrun = NOT_RUN
+ try:
+ self.generator = kw['generator']
+ except KeyError:
+ self.generator = self
+
+ self.env = kw['env']
+ """:py:class:`waflib.ConfigSet.ConfigSet` object (make sure to provide one)"""
+
+ self.inputs = []
+ """List of input nodes, which represent the files used by the task instance"""
+
+ self.outputs = []
+ """List of output nodes, which represent the files created by the task instance"""
+
+ self.dep_nodes = []
+ """List of additional nodes to depend on"""
+
+ self.run_after = set()
+ """Set of tasks that must be executed before this one"""
+
+ def __lt__(self, other):
+ return self.priority() > other.priority()
+ def __le__(self, other):
+ return self.priority() >= other.priority()
+ def __gt__(self, other):
+ return self.priority() < other.priority()
+ def __ge__(self, other):
+ return self.priority() <= other.priority()
+
+ def get_cwd(self):
+ """
+ :return: current working directory
+ :rtype: :py:class:`waflib.Node.Node`
+ """
+ bld = self.generator.bld
+ ret = getattr(self, 'cwd', None) or getattr(bld, 'cwd', bld.bldnode)
+ if isinstance(ret, str):
+ if os.path.isabs(ret):
+ ret = bld.root.make_node(ret)
+ else:
+ ret = self.generator.path.make_node(ret)
+ return ret
+
+ def quote_flag(self, x):
+ """
+ Surround a process argument by quotes so that a list of arguments can be written to a file
+
+ :param x: flag
+ :type x: string
+ :return: quoted flag
+ :rtype: string
+ """
+ old = x
+ if '\\' in x:
+ x = x.replace('\\', '\\\\')
+ if '"' in x:
+ x = x.replace('"', '\\"')
+ if old != x or ' ' in x or '\t' in x or "'" in x:
+ x = '"%s"' % x
+ return x
+
+ def priority(self):
+ """
+ Priority of execution; the higher, the earlier
+
+ :return: the priority value
+ :rtype: a tuple of numeric values
+ """
+ return (self.weight + self.prio_order, - getattr(self.generator, 'tg_idx_count', 0))
+
+ def split_argfile(self, cmd):
+ """
+ Splits a list of process commands into the executable part and its list of arguments
+
+ :return: a tuple containing the executable first and then the rest of arguments
+ :rtype: tuple
+ """
+ return ([cmd[0]], [self.quote_flag(x) for x in cmd[1:]])
+
+ def exec_command(self, cmd, **kw):
+ """
+ Wrapper for :py:meth:`waflib.Context.Context.exec_command`.
+ This version set the current working directory (``build.variant_dir``),
+ applies PATH settings (if self.env.PATH is provided), and can run long
+ commands through a temporary ``@argfile``.
+
+ :param cmd: process command to execute
+ :type cmd: list of string (best) or string (process will use a shell)
+ :return: the return code
+ :rtype: int
+
+ Optional parameters:
+
+ #. cwd: current working directory (Node or string)
+ #. stdout: set to None to prevent waf from capturing the process standard output
+ #. stderr: set to None to prevent waf from capturing the process standard error
+ #. timeout: timeout value (Python 3)
+ """
+ if not 'cwd' in kw:
+ kw['cwd'] = self.get_cwd()
+
+ if hasattr(self, 'timeout'):
+ kw['timeout'] = self.timeout
+
+ if self.env.PATH:
+ env = kw['env'] = dict(kw.get('env') or self.env.env or os.environ)
+ env['PATH'] = self.env.PATH if isinstance(self.env.PATH, str) else os.pathsep.join(self.env.PATH)
+
+ if hasattr(self, 'stdout'):
+ kw['stdout'] = self.stdout
+ if hasattr(self, 'stderr'):
+ kw['stderr'] = self.stderr
+
+ # workaround for command line length limit:
+ # http://support.microsoft.com/kb/830473
+ if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000):
+ cmd, args = self.split_argfile(cmd)
+ try:
+ (fd, tmp) = tempfile.mkstemp()
+ os.write(fd, '\r\n'.join(args).encode())
+ os.close(fd)
+ if Logs.verbose:
+ Logs.debug('argfile: @%r -> %r', tmp, args)
+ return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw)
+ finally:
+ try:
+ os.remove(tmp)
+ except OSError:
+ # anti-virus and indexers can keep files open -_-
+ pass
+ else:
+ return self.generator.bld.exec_command(cmd, **kw)
+
+ def process(self):
+ """
+ Runs the task and handles errors
+
+ :return: 0 or None if everything is fine
+ :rtype: integer
+ """
+ # remove the task signature immediately before it is executed
+ # so that the task will be executed again in case of failure
+ try:
+ del self.generator.bld.task_sigs[self.uid()]
+ except KeyError:
+ pass
+
+ try:
+ ret = self.run()
+ except Exception:
+ self.err_msg = traceback.format_exc()
+ self.hasrun = EXCEPTION
+ else:
+ if ret:
+ self.err_code = ret
+ self.hasrun = CRASHED
+ else:
+ try:
+ self.post_run()
+ except Errors.WafError:
+ pass
+ except Exception:
+ self.err_msg = traceback.format_exc()
+ self.hasrun = EXCEPTION
+ else:
+ self.hasrun = SUCCESS
+
+ if self.hasrun != SUCCESS and self.scan:
+ # rescan dependencies on next run
+ try:
+ del self.generator.bld.imp_sigs[self.uid()]
+ except KeyError:
+ pass
+
+ def log_display(self, bld):
+ "Writes the execution status on the context logger"
+ if self.generator.bld.progress_bar == 3:
+ return
+
+ s = self.display()
+ if s:
+ if bld.logger:
+ logger = bld.logger
+ else:
+ logger = Logs
+
+ if self.generator.bld.progress_bar == 1:
+ c1 = Logs.colors.cursor_off
+ c2 = Logs.colors.cursor_on
+ logger.info(s, extra={'stream': sys.stderr, 'terminator':'', 'c1': c1, 'c2' : c2})
+ else:
+ logger.info(s, extra={'terminator':'', 'c1': '', 'c2' : ''})
+
+ def display(self):
+ """
+ Returns an execution status for the console, the progress bar, or the IDE output.
+
+ :rtype: string
+ """
+ col1 = Logs.colors(self.color)
+ col2 = Logs.colors.NORMAL
+ master = self.generator.bld.producer
+
+ def cur():
+ # the current task position, computed as late as possible
+ return master.processed - master.ready.qsize()
+
+ if self.generator.bld.progress_bar == 1:
+ return self.generator.bld.progress_line(cur(), master.total, col1, col2)
+
+ if self.generator.bld.progress_bar == 2:
+ ela = str(self.generator.bld.timer)
+ try:
+ ins = ','.join([n.name for n in self.inputs])
+ except AttributeError:
+ ins = ''
+ try:
+ outs = ','.join([n.name for n in self.outputs])
+ except AttributeError:
+ outs = ''
+ return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (master.total, cur(), ins, outs, ela)
+
+ s = str(self)
+ if not s:
+ return None
+
+ total = master.total
+ n = len(str(total))
+ fs = '[%%%dd/%%%dd] %%s%%s%%s%%s\n' % (n, n)
+ kw = self.keyword()
+ if kw:
+ kw += ' '
+ return fs % (cur(), total, kw, col1, s, col2)
+
+ def hash_constraints(self):
+ """
+ Identifies a task type for all the constraints relevant for the scheduler: precedence, file production
+
+ :return: a hash value
+ :rtype: string
+ """
+ return (tuple(self.before), tuple(self.after), tuple(self.ext_in), tuple(self.ext_out), self.__class__.__name__, self.hcode)
+
+ def format_error(self):
+ """
+ Returns an error message to display the build failure reasons
+
+ :rtype: string
+ """
+ if Logs.verbose:
+ msg = ': %r\n%r' % (self, getattr(self, 'last_cmd', ''))
+ else:
+ msg = ' (run with -v to display more information)'
+ name = getattr(self.generator, 'name', '')
+ if getattr(self, "err_msg", None):
+ return self.err_msg
+ elif not self.hasrun:
+ return 'task in %r was not executed for some reason: %r' % (name, self)
+ elif self.hasrun == CRASHED:
+ try:
+ return ' -> task in %r failed with exit status %r%s' % (name, self.err_code, msg)
+ except AttributeError:
+ return ' -> task in %r failed%s' % (name, msg)
+ elif self.hasrun == MISSING:
+ return ' -> missing files in %r%s' % (name, msg)
+ elif self.hasrun == CANCELED:
+ return ' -> %r canceled because of missing dependencies' % name
+ else:
+ return 'invalid status for task in %r: %r' % (name, self.hasrun)
+
+ def colon(self, var1, var2):
+ """
+ Enable scriptlet expressions of the form ${FOO_ST:FOO}
+ If the first variable (FOO_ST) is empty, then an empty list is returned
+
+ The results will be slightly different if FOO_ST is a list, for example::
+
+ env.FOO = ['p1', 'p2']
+ env.FOO_ST = '-I%s'
+ # ${FOO_ST:FOO} returns
+ ['-Ip1', '-Ip2']
+
+ env.FOO_ST = ['-a', '-b']
+ # ${FOO_ST:FOO} returns
+ ['-a', '-b', 'p1', '-a', '-b', 'p2']
+ """
+ tmp = self.env[var1]
+ if not tmp:
+ return []
+
+ if isinstance(var2, str):
+ it = self.env[var2]
+ else:
+ it = var2
+ if isinstance(tmp, str):
+ return [tmp % x for x in it]
+ else:
+ lst = []
+ for y in it:
+ lst.extend(tmp)
+ lst.append(y)
+ return lst
+
+ def __str__(self):
+ "string to display to the user"
+ name = self.__class__.__name__
+ if self.outputs:
+ if name.endswith(('lib', 'program')) or not self.inputs:
+ node = self.outputs[0]
+ return node.path_from(node.ctx.launch_node())
+ if not (self.inputs or self.outputs):
+ return self.__class__.__name__
+ if len(self.inputs) == 1:
+ node = self.inputs[0]
+ return node.path_from(node.ctx.launch_node())
+
+ src_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.inputs])
+ tgt_str = ' '.join([a.path_from(a.ctx.launch_node()) for a in self.outputs])
+ if self.outputs:
+ sep = ' -> '
+ else:
+ sep = ''
+ return '%s: %s%s%s' % (self.__class__.__name__, src_str, sep, tgt_str)
+
+ def keyword(self):
+ "Display keyword used to prettify the console outputs"
+ name = self.__class__.__name__
+ if name.endswith(('lib', 'program')):
+ return 'Linking'
+ if len(self.inputs) == 1 and len(self.outputs) == 1:
+ return 'Compiling'
+ if not self.inputs:
+ if self.outputs:
+ return 'Creating'
+ else:
+ return 'Running'
+ return 'Processing'
+
+ def __repr__(self):
+ "for debugging purposes"
+ try:
+ ins = ",".join([x.name for x in self.inputs])
+ outs = ",".join([x.name for x in self.outputs])
+ except AttributeError:
+ ins = ",".join([str(x) for x in self.inputs])
+ outs = ",".join([str(x) for x in self.outputs])
+ return "".join(['\n\t{task %r: ' % id(self), self.__class__.__name__, " ", ins, " -> ", outs, '}'])
+
+ def uid(self):
+ """
+ Returns an identifier used to determine if tasks are up-to-date. Since the
+ identifier will be stored between executions, it must be:
+
+ - unique for a task: no two tasks return the same value (for a given build context)
+ - the same for a given task instance
+
+ By default, the node paths, the class name, and the function are used
+ as inputs to compute a hash.
+
+ The pointer to the object (python built-in 'id') will change between build executions,
+ and must be avoided in such hashes.
+
+ :return: hash value
+ :rtype: string
+ """
+ try:
+ return self.uid_
+ except AttributeError:
+ m = Utils.md5(self.__class__.__name__)
+ up = m.update
+ for x in self.inputs + self.outputs:
+ up(x.abspath())
+ self.uid_ = m.digest()
+ return self.uid_
+
+ def set_inputs(self, inp):
+ """
+ Appends the nodes to the *inputs* list
+
+ :param inp: input nodes
+ :type inp: node or list of nodes
+ """
+ if isinstance(inp, list):
+ self.inputs += inp
+ else:
+ self.inputs.append(inp)
+
+ def set_outputs(self, out):
+ """
+ Appends the nodes to the *outputs* list
+
+ :param out: output nodes
+ :type out: node or list of nodes
+ """
+ if isinstance(out, list):
+ self.outputs += out
+ else:
+ self.outputs.append(out)
+
+ def set_run_after(self, task):
+ """
+ Run this task only after the given *task*.
+
+ Calling this method from :py:meth:`waflib.Task.Task.runnable_status` may cause
+ build deadlocks; see :py:meth:`waflib.Tools.fc.fc.runnable_status` for details.
+
+ :param task: task
+ :type task: :py:class:`waflib.Task.Task`
+ """
+ assert isinstance(task, Task)
+ self.run_after.add(task)
+
+ def signature(self):
+ """
+ Task signatures are stored between build executions, they are use to track the changes
+ made to the input nodes (not to the outputs!). The signature hashes data from various sources:
+
+ * explicit dependencies: files listed in the inputs (list of node objects) :py:meth:`waflib.Task.Task.sig_explicit_deps`
+ * implicit dependencies: list of nodes returned by scanner methods (when present) :py:meth:`waflib.Task.Task.sig_implicit_deps`
+ * hashed data: variables/values read from task.vars/task.env :py:meth:`waflib.Task.Task.sig_vars`
+
+ If the signature is expected to give a different result, clear the cache kept in ``self.cache_sig``::
+
+ from waflib import Task
+ class cls(Task.Task):
+ def signature(self):
+ sig = super(Task.Task, self).signature()
+ delattr(self, 'cache_sig')
+ return super(Task.Task, self).signature()
+
+ :return: the signature value
+ :rtype: string or bytes
+ """
+ try:
+ return self.cache_sig
+ except AttributeError:
+ pass
+
+ self.m = Utils.md5(self.hcode)
+
+ # explicit deps
+ self.sig_explicit_deps()
+
+ # env vars
+ self.sig_vars()
+
+ # implicit deps / scanner results
+ if self.scan:
+ try:
+ self.sig_implicit_deps()
+ except Errors.TaskRescan:
+ return self.signature()
+
+ ret = self.cache_sig = self.m.digest()
+ return ret
+
+ def runnable_status(self):
+ """
+ Returns the Task status
+
+ :return: a task state in :py:const:`waflib.Task.RUN_ME`,
+ :py:const:`waflib.Task.SKIP_ME`, :py:const:`waflib.Task.CANCEL_ME` or :py:const:`waflib.Task.ASK_LATER`.
+ :rtype: int
+ """
+ bld = self.generator.bld
+ if bld.is_install < 0:
+ return SKIP_ME
+
+ for t in self.run_after:
+ if not t.hasrun:
+ return ASK_LATER
+ elif t.hasrun < SKIPPED:
+ # a dependency has an error
+ return CANCEL_ME
+
+ # first compute the signature
+ try:
+ new_sig = self.signature()
+ except Errors.TaskNotReady:
+ return ASK_LATER
+
+ # compare the signature to a signature computed previously
+ key = self.uid()
+ try:
+ prev_sig = bld.task_sigs[key]
+ except KeyError:
+ Logs.debug('task: task %r must run: it was never run before or the task code changed', self)
+ return RUN_ME
+
+ if new_sig != prev_sig:
+ Logs.debug('task: task %r must run: the task signature changed', self)
+ return RUN_ME
+
+ # compare the signatures of the outputs
+ for node in self.outputs:
+ sig = bld.node_sigs.get(node)
+ if not sig:
+ Logs.debug('task: task %r must run: an output node has no signature', self)
+ return RUN_ME
+ if sig != key:
+ Logs.debug('task: task %r must run: an output node was produced by another task', self)
+ return RUN_ME
+ if not node.exists():
+ Logs.debug('task: task %r must run: an output node does not exist', self)
+ return RUN_ME
+
+ return (self.always_run and RUN_ME) or SKIP_ME
+
+ def post_run(self):
+ """
+ Called after successful execution to record that the task has run by
+ updating the entry in :py:attr:`waflib.Build.BuildContext.task_sigs`.
+ """
+ bld = self.generator.bld
+ for node in self.outputs:
+ if not node.exists():
+ self.hasrun = MISSING
+ self.err_msg = '-> missing file: %r' % node.abspath()
+ raise Errors.WafError(self.err_msg)
+ bld.node_sigs[node] = self.uid() # make sure this task produced the files in question
+ bld.task_sigs[self.uid()] = self.signature()
+ if not self.keep_last_cmd:
+ try:
+ del self.last_cmd
+ except AttributeError:
+ pass
+
+ def sig_explicit_deps(self):
+ """
+ Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.inputs`
+ and :py:attr:`waflib.Task.Task.dep_nodes` signatures.
+ """
+ bld = self.generator.bld
+ upd = self.m.update
+
+ # the inputs
+ for x in self.inputs + self.dep_nodes:
+ upd(x.get_bld_sig())
+
+ # manual dependencies, they can slow down the builds
+ if bld.deps_man:
+ additional_deps = bld.deps_man
+ for x in self.inputs + self.outputs:
+ try:
+ d = additional_deps[x]
+ except KeyError:
+ continue
+
+ for v in d:
+ try:
+ v = v.get_bld_sig()
+ except AttributeError:
+ if hasattr(v, '__call__'):
+ v = v() # dependency is a function, call it
+ upd(v)
+
+ def sig_deep_inputs(self):
+ """
+ Enable rebuilds on input files task signatures. Not used by default.
+
+ Example: hashes of output programs can be unchanged after being re-linked,
+ despite the libraries being different. This method can thus prevent stale unit test
+ results (waf_unit_test.py).
+
+ Hashing input file timestamps is another possibility for the implementation.
+ This may cause unnecessary rebuilds when input tasks are frequently executed.
+ Here is an implementation example::
+
+ lst = []
+ for node in self.inputs + self.dep_nodes:
+ st = os.stat(node.abspath())
+ lst.append(st.st_mtime)
+ lst.append(st.st_size)
+ self.m.update(Utils.h_list(lst))
+
+ The downside of the implementation is that it absolutely requires all build directory
+ files to be declared within the current build.
+ """
+ bld = self.generator.bld
+ lst = [bld.task_sigs[bld.node_sigs[node]] for node in (self.inputs + self.dep_nodes) if node.is_bld()]
+ self.m.update(Utils.h_list(lst))
+
+ def sig_vars(self):
+ """
+ Used by :py:meth:`waflib.Task.Task.signature`; it hashes :py:attr:`waflib.Task.Task.env` variables/values
+ When overriding this method, and if scriptlet expressions are used, make sure to follow
+ the code in :py:meth:`waflib.Task.Task.compile_sig_vars` to enable dependencies on scriptlet results.
+ """
+ sig = self.generator.bld.hash_env_vars(self.env, self.vars)
+ self.m.update(sig)
+
+ scan = None
+ """
+ This method, when provided, returns a tuple containing:
+
+ * a list of nodes corresponding to real files
+ * a list of names for files not found in path_lst
+
+ For example::
+
+ from waflib.Task import Task
+ class mytask(Task):
+ def scan(self, node):
+ return ([], [])
+
+ The first and second lists in the tuple are stored in :py:attr:`waflib.Build.BuildContext.node_deps` and
+ :py:attr:`waflib.Build.BuildContext.raw_deps` respectively.
+ """
+
+ def sig_implicit_deps(self):
+ """
+ Used by :py:meth:`waflib.Task.Task.signature`; it hashes node signatures
+ obtained by scanning for dependencies (:py:meth:`waflib.Task.Task.scan`).
+
+ The exception :py:class:`waflib.Errors.TaskRescan` is thrown
+ when a file has changed. In this case, the method :py:meth:`waflib.Task.Task.signature` is called
+ once again, and return here to call :py:meth:`waflib.Task.Task.scan` and searching for dependencies.
+ """
+ bld = self.generator.bld
+
+ # get the task signatures from previous runs
+ key = self.uid()
+ prev = bld.imp_sigs.get(key, [])
+
+ # for issue #379
+ if prev:
+ try:
+ if prev == self.compute_sig_implicit_deps():
+ return prev
+ except Errors.TaskNotReady:
+ raise
+ except EnvironmentError:
+ # when a file was renamed, remove the stale nodes (headers in folders without source files)
+ # this will break the order calculation for headers created during the build in the source directory (should be uncommon)
+ # the behaviour will differ when top != out
+ for x in bld.node_deps.get(self.uid(), []):
+ if not x.is_bld() and not x.exists():
+ try:
+ del x.parent.children[x.name]
+ except KeyError:
+ pass
+ del bld.imp_sigs[key]
+ raise Errors.TaskRescan('rescan')
+
+ # no previous run or the signature of the dependencies has changed, rescan the dependencies
+ (bld.node_deps[key], bld.raw_deps[key]) = self.scan()
+ if Logs.verbose:
+ Logs.debug('deps: scanner for %s: %r; unresolved: %r', self, bld.node_deps[key], bld.raw_deps[key])
+
+ # recompute the signature and return it
+ try:
+ bld.imp_sigs[key] = self.compute_sig_implicit_deps()
+ except EnvironmentError:
+ for k in bld.node_deps.get(self.uid(), []):
+ if not k.exists():
+ Logs.warn('Dependency %r for %r is missing: check the task declaration and the build order!', k, self)
+ raise
+
+ def compute_sig_implicit_deps(self):
+ """
+ Used by :py:meth:`waflib.Task.Task.sig_implicit_deps` for computing the actual hash of the
+ :py:class:`waflib.Node.Node` returned by the scanner.
+
+ :return: a hash value for the implicit dependencies
+ :rtype: string or bytes
+ """
+ upd = self.m.update
+ self.are_implicit_nodes_ready()
+
+ # scanner returns a node that does not have a signature
+ # just *ignore* the error and let them figure out from the compiler output
+ # waf -k behaviour
+ for k in self.generator.bld.node_deps.get(self.uid(), []):
+ upd(k.get_bld_sig())
+ return self.m.digest()
+
+ def are_implicit_nodes_ready(self):
+ """
+ For each node returned by the scanner, see if there is a task that creates it,
+ and infer the build order
+
+ This has a low performance impact on null builds (1.86s->1.66s) thanks to caching (28s->1.86s)
+ """
+ bld = self.generator.bld
+ try:
+ cache = bld.dct_implicit_nodes
+ except AttributeError:
+ bld.dct_implicit_nodes = cache = {}
+
+ # one cache per build group
+ try:
+ dct = cache[bld.current_group]
+ except KeyError:
+ dct = cache[bld.current_group] = {}
+ for tsk in bld.cur_tasks:
+ for x in tsk.outputs:
+ dct[x] = tsk
+
+ modified = False
+ for x in bld.node_deps.get(self.uid(), []):
+ if x in dct:
+ self.run_after.add(dct[x])
+ modified = True
+
+ if modified:
+ for tsk in self.run_after:
+ if not tsk.hasrun:
+ #print "task is not ready..."
+ raise Errors.TaskNotReady('not ready')
+if sys.hexversion > 0x3000000:
+ def uid(self):
+ try:
+ return self.uid_
+ except AttributeError:
+ m = Utils.md5(self.__class__.__name__.encode('latin-1', 'xmlcharrefreplace'))
+ up = m.update
+ for x in self.inputs + self.outputs:
+ up(x.abspath().encode('latin-1', 'xmlcharrefreplace'))
+ self.uid_ = m.digest()
+ return self.uid_
+ uid.__doc__ = Task.uid.__doc__
+ Task.uid = uid
+
+def is_before(t1, t2):
+ """
+ Returns a non-zero value if task t1 is to be executed before task t2::
+
+ t1.ext_out = '.h'
+ t2.ext_in = '.h'
+ t2.after = ['t1']
+ t1.before = ['t2']
+ waflib.Task.is_before(t1, t2) # True
+
+ :param t1: Task object
+ :type t1: :py:class:`waflib.Task.Task`
+ :param t2: Task object
+ :type t2: :py:class:`waflib.Task.Task`
+ """
+ to_list = Utils.to_list
+ for k in to_list(t2.ext_in):
+ if k in to_list(t1.ext_out):
+ return 1
+
+ if t1.__class__.__name__ in to_list(t2.after):
+ return 1
+
+ if t2.__class__.__name__ in to_list(t1.before):
+ return 1
+
+ return 0
+
+def set_file_constraints(tasks):
+ """
+ Updates the ``run_after`` attribute of all tasks based on the task inputs and outputs
+
+ :param tasks: tasks
+ :type tasks: list of :py:class:`waflib.Task.Task`
+ """
+ ins = Utils.defaultdict(set)
+ outs = Utils.defaultdict(set)
+ for x in tasks:
+ for a in x.inputs:
+ ins[a].add(x)
+ for a in x.dep_nodes:
+ ins[a].add(x)
+ for a in x.outputs:
+ outs[a].add(x)
+
+ links = set(ins.keys()).intersection(outs.keys())
+ for k in links:
+ for a in ins[k]:
+ a.run_after.update(outs[k])
+
+
+class TaskGroup(object):
+ """
+ Wrap nxm task order constraints into a single object
+ to prevent the creation of large list/set objects
+
+ This is an optimization
+ """
+ def __init__(self, prev, next):
+ self.prev = prev
+ self.next = next
+ self.done = False
+
+ def get_hasrun(self):
+ for k in self.prev:
+ if not k.hasrun:
+ return NOT_RUN
+ return SUCCESS
+
+ hasrun = property(get_hasrun, None)
+
+def set_precedence_constraints(tasks):
+ """
+ Updates the ``run_after`` attribute of all tasks based on the after/before/ext_out/ext_in attributes
+
+ :param tasks: tasks
+ :type tasks: list of :py:class:`waflib.Task.Task`
+ """
+ cstr_groups = Utils.defaultdict(list)
+ for x in tasks:
+ h = x.hash_constraints()
+ cstr_groups[h].append(x)
+
+ keys = list(cstr_groups.keys())
+ maxi = len(keys)
+
+ # this list should be short
+ for i in range(maxi):
+ t1 = cstr_groups[keys[i]][0]
+ for j in range(i + 1, maxi):
+ t2 = cstr_groups[keys[j]][0]
+
+ # add the constraints based on the comparisons
+ if is_before(t1, t2):
+ a = i
+ b = j
+ elif is_before(t2, t1):
+ a = j
+ b = i
+ else:
+ continue
+
+ a = cstr_groups[keys[a]]
+ b = cstr_groups[keys[b]]
+
+ if len(a) < 2 or len(b) < 2:
+ for x in b:
+ x.run_after.update(a)
+ else:
+ group = TaskGroup(set(a), set(b))
+ for x in b:
+ x.run_after.add(group)
+
+def funex(c):
+ """
+ Compiles a scriptlet expression into a Python function
+
+ :param c: function to compile
+ :type c: string
+ :return: the function 'f' declared in the input string
+ :rtype: function
+ """
+ dc = {}
+ exec(c, dc)
+ return dc['f']
+
+re_cond = re.compile('(?P<var>\w+)|(?P<or>\|)|(?P<and>&)')
+re_novar = re.compile(r'^(SRC|TGT)\W+.*?$')
+reg_act = re.compile(r'(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})', re.M)
+def compile_fun_shell(line):
+ """
+ Creates a compiled function to execute a process through a sub-shell
+ """
+ extr = []
+ def repl(match):
+ g = match.group
+ if g('dollar'):
+ return "$"
+ elif g('backslash'):
+ return '\\\\'
+ elif g('subst'):
+ extr.append((g('var'), g('code')))
+ return "%s"
+ return None
+ line = reg_act.sub(repl, line) or line
+ dvars = []
+ def add_dvar(x):
+ if x not in dvars:
+ dvars.append(x)
+
+ def replc(m):
+ # performs substitutions and populates dvars
+ if m.group('and'):
+ return ' and '
+ elif m.group('or'):
+ return ' or '
+ else:
+ x = m.group('var')
+ add_dvar(x)
+ return 'env[%r]' % x
+
+ parm = []
+ app = parm.append
+ for (var, meth) in extr:
+ if var == 'SRC':
+ if meth:
+ app('tsk.inputs%s' % meth)
+ else:
+ app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
+ elif var == 'TGT':
+ if meth:
+ app('tsk.outputs%s' % meth)
+ else:
+ app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
+ elif meth:
+ if meth.startswith(':'):
+ add_dvar(var)
+ m = meth[1:]
+ if m == 'SRC':
+ m = '[a.path_from(cwdx) for a in tsk.inputs]'
+ elif m == 'TGT':
+ m = '[a.path_from(cwdx) for a in tsk.outputs]'
+ elif re_novar.match(m):
+ m = '[tsk.inputs%s]' % m[3:]
+ elif re_novar.match(m):
+ m = '[tsk.outputs%s]' % m[3:]
+ else:
+ add_dvar(m)
+ if m[:3] not in ('tsk', 'gen', 'bld'):
+ m = '%r' % m
+ app('" ".join(tsk.colon(%r, %s))' % (var, m))
+ elif meth.startswith('?'):
+ # In A?B|C output env.A if one of env.B or env.C is non-empty
+ expr = re_cond.sub(replc, meth[1:])
+ app('p(%r) if (%s) else ""' % (var, expr))
+ else:
+ call = '%s%s' % (var, meth)
+ add_dvar(call)
+ app(call)
+ else:
+ add_dvar(var)
+ app("p('%s')" % var)
+ if parm:
+ parm = "%% (%s) " % (',\n\t\t'.join(parm))
+ else:
+ parm = ''
+
+ c = COMPILE_TEMPLATE_SHELL % (line, parm)
+ Logs.debug('action: %s', c.strip().splitlines())
+ return (funex(c), dvars)
+
+reg_act_noshell = re.compile(r"(?P<space>\s+)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})|(?P<text>([^$ \t\n\r\f\v]|\$\$)+)", re.M)
+def compile_fun_noshell(line):
+ """
+ Creates a compiled function to execute a process without a sub-shell
+ """
+ buf = []
+ dvars = []
+ merge = False
+ app = buf.append
+
+ def add_dvar(x):
+ if x not in dvars:
+ dvars.append(x)
+
+ def replc(m):
+ # performs substitutions and populates dvars
+ if m.group('and'):
+ return ' and '
+ elif m.group('or'):
+ return ' or '
+ else:
+ x = m.group('var')
+ add_dvar(x)
+ return 'env[%r]' % x
+
+ for m in reg_act_noshell.finditer(line):
+ if m.group('space'):
+ merge = False
+ continue
+ elif m.group('text'):
+ app('[%r]' % m.group('text').replace('$$', '$'))
+ elif m.group('subst'):
+ var = m.group('var')
+ code = m.group('code')
+ if var == 'SRC':
+ if code:
+ app('[tsk.inputs%s]' % code)
+ else:
+ app('[a.path_from(cwdx) for a in tsk.inputs]')
+ elif var == 'TGT':
+ if code:
+ app('[tsk.outputs%s]' % code)
+ else:
+ app('[a.path_from(cwdx) for a in tsk.outputs]')
+ elif code:
+ if code.startswith(':'):
+ # a composed variable ${FOO:OUT}
+ add_dvar(var)
+ m = code[1:]
+ if m == 'SRC':
+ m = '[a.path_from(cwdx) for a in tsk.inputs]'
+ elif m == 'TGT':
+ m = '[a.path_from(cwdx) for a in tsk.outputs]'
+ elif re_novar.match(m):
+ m = '[tsk.inputs%s]' % m[3:]
+ elif re_novar.match(m):
+ m = '[tsk.outputs%s]' % m[3:]
+ else:
+ add_dvar(m)
+ if m[:3] not in ('tsk', 'gen', 'bld'):
+ m = '%r' % m
+ app('tsk.colon(%r, %s)' % (var, m))
+ elif code.startswith('?'):
+ # In A?B|C output env.A if one of env.B or env.C is non-empty
+ expr = re_cond.sub(replc, code[1:])
+ app('to_list(env[%r] if (%s) else [])' % (var, expr))
+ else:
+ # plain code such as ${tsk.inputs[0].abspath()}
+ call = '%s%s' % (var, code)
+ add_dvar(call)
+ app('gen.to_list(%s)' % call)
+ else:
+ # a plain variable such as # a plain variable like ${AR}
+ app('to_list(env[%r])' % var)
+ add_dvar(var)
+ if merge:
+ tmp = 'merge(%s, %s)' % (buf[-2], buf[-1])
+ del buf[-1]
+ buf[-1] = tmp
+ merge = True # next turn
+
+ buf = ['lst.extend(%s)' % x for x in buf]
+ fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf)
+ Logs.debug('action: %s', fun.strip().splitlines())
+ return (funex(fun), dvars)
+
+def compile_fun(line, shell=False):
+ """
+ Parses a string expression such as '${CC} ${SRC} -o ${TGT}' and returns a pair containing:
+
+ * The function created (compiled) for use as :py:meth:`waflib.Task.Task.run`
+ * The list of variables that must cause rebuilds when *env* data is modified
+
+ for example::
+
+ from waflib.Task import compile_fun
+ compile_fun('cxx', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}')
+
+ def build(bld):
+ bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"')
+
+ The env variables (CXX, ..) on the task must not hold dicts so as to preserve a consistent order.
+ The reserved keywords ``TGT`` and ``SRC`` represent the task input and output nodes
+
+ """
+ if isinstance(line, str):
+ if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
+ shell = True
+ else:
+ dvars_lst = []
+ funs_lst = []
+ for x in line:
+ if isinstance(x, str):
+ fun, dvars = compile_fun(x, shell)
+ dvars_lst += dvars
+ funs_lst.append(fun)
+ else:
+ # assume a function to let through
+ funs_lst.append(x)
+ def composed_fun(task):
+ for x in funs_lst:
+ ret = x(task)
+ if ret:
+ return ret
+ return None
+ return composed_fun, dvars_lst
+ if shell:
+ return compile_fun_shell(line)
+ else:
+ return compile_fun_noshell(line)
+
+def compile_sig_vars(vars):
+ """
+ This method produces a sig_vars method suitable for subclasses that provide
+ scriptlet code in their run_str code.
+ If no such method can be created, this method returns None.
+
+ The purpose of the sig_vars method returned is to ensures
+ that rebuilds occur whenever the contents of the expression changes.
+ This is the case B below::
+
+ import time
+ # case A: regular variables
+ tg = bld(rule='echo ${FOO}')
+ tg.env.FOO = '%s' % time.time()
+ # case B
+ bld(rule='echo ${gen.foo}', foo='%s' % time.time())
+
+ :param vars: env variables such as CXXFLAGS or gen.foo
+ :type vars: list of string
+ :return: A sig_vars method relevant for dependencies if adequate, else None
+ :rtype: A function, or None in most cases
+ """
+ buf = []
+ for x in sorted(vars):
+ if x[:3] in ('tsk', 'gen', 'bld'):
+ buf.append('buf.append(%s)' % x)
+ if buf:
+ return funex(COMPILE_TEMPLATE_SIG_VARS % '\n\t'.join(buf))
+ return None
+
+def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[], before=[], after=[], shell=False, scan=None):
+ """
+ Returns a new task subclass with the function ``run`` compiled from the line given.
+
+ :param func: method run
+ :type func: string or function
+ :param vars: list of variables to hash
+ :type vars: list of string
+ :param color: color to use
+ :type color: string
+ :param shell: when *func* is a string, enable/disable the use of the shell
+ :type shell: bool
+ :param scan: method scan
+ :type scan: function
+ :rtype: :py:class:`waflib.Task.Task`
+ """
+
+ params = {
+ 'vars': vars or [], # function arguments are static, and this one may be modified by the class
+ 'color': color,
+ 'name': name,
+ 'shell': shell,
+ 'scan': scan,
+ }
+
+ if isinstance(func, str) or isinstance(func, tuple):
+ params['run_str'] = func
+ else:
+ params['run'] = func
+
+ cls = type(Task)(name, (Task,), params)
+ classes[name] = cls
+
+ if ext_in:
+ cls.ext_in = Utils.to_list(ext_in)
+ if ext_out:
+ cls.ext_out = Utils.to_list(ext_out)
+ if before:
+ cls.before = Utils.to_list(before)
+ if after:
+ cls.after = Utils.to_list(after)
+
+ return cls
+
+def deep_inputs(cls):
+ """
+ Task class decorator to enable rebuilds on input files task signatures
+ """
+ def sig_explicit_deps(self):
+ Task.sig_explicit_deps(self)
+ Task.sig_deep_inputs(self)
+ cls.sig_explicit_deps = sig_explicit_deps
+ return cls
+
+TaskBase = Task
+"Provided for compatibility reasons, TaskBase should not be used"
+
+class TaskSemaphore(object):
+ """
+ Task semaphores provide a simple and efficient way of throttling the amount of
+ a particular task to run concurrently. The throttling value is capped
+ by the amount of maximum jobs, so for example, a `TaskSemaphore(10)`
+ has no effect in a `-j2` build.
+
+ Task semaphores are typically specified on the task class level::
+
+ class compile(waflib.Task.Task):
+ semaphore = waflib.Task.TaskSemaphore(2)
+ run_str = 'touch ${TGT}'
+
+ Task semaphores are meant to be used by the build scheduler in the main
+ thread, so there are no guarantees of thread safety.
+ """
+ def __init__(self, num):
+ """
+ :param num: maximum value of concurrent tasks
+ :type num: int
+ """
+ self.num = num
+ self.locking = set()
+ self.waiting = set()
+
+ def is_locked(self):
+ """Returns True if this semaphore cannot be acquired by more tasks"""
+ return len(self.locking) >= self.num
+
+ def acquire(self, tsk):
+ """
+ Mark the semaphore as used by the given task (not re-entrant).
+
+ :param tsk: task object
+ :type tsk: :py:class:`waflib.Task.Task`
+ :raises: :py:class:`IndexError` in case the resource is already acquired
+ """
+ if self.is_locked():
+ raise IndexError('Cannot lock more %r' % self.locking)
+ self.locking.add(tsk)
+
+ def release(self, tsk):
+ """
+ Mark the semaphore as unused by the given task.
+
+ :param tsk: task object
+ :type tsk: :py:class:`waflib.Task.Task`
+ :raises: :py:class:`KeyError` in case the resource is not acquired by the task
+ """
+ self.locking.remove(tsk)
+