A compiler producing source files with names unknown in advance

The example below demonstrates how to tackle the following requirements:

The main difficulty in this scenario is to store the information on the source file produced and to create the corresponding tasks each time.


## A DEMO ##

VERSION='0.0.1'
APPNAME='unknown_outputs'
srcdir = '.'
blddir = 'build'

def set_options(opt):
	pass

def configure(conf):
	# used only when configured from the same folder
	conf.check_tool('gcc')
	conf.env["SHPIP_COMPILER"] = os.getcwd() + os.sep + "bad_compiler.py"

def build(bld):
	staticlib = bld.new_task_gen()
	staticlib.features = 'cc cstaticlib'
	staticlib.source = 'x.c foo.shpip' 1
	staticlib.target='teststaticlib'
	staticlib.includes = '.'


## INTERNAL CODE BELOW ##

import os
import TaskGen, Task, Utils, Build
from TaskGen import taskgen, feature, before, after, extension
from logging import debug
from Constants import *

@taskgen
@after('apply_link')
@extension('.shpip')
def process_shpip(self, node): 2
	tsk = shpip_task(self.env, generator=self)
	tsk.task_gen = self
	tsk.set_inputs(node)

class shpip_task(Task.Task): 3
	"""
	A special task, which finds its outputs once it has run
	It outputs cpp files that must be compiled too
	"""

	color = 'PINK'
	quiet = 1 4

	# important, no link before all shpip are done
	before = ['cc_link', 'cxx_link', 'ar_link_static']

	def __init__(self, *k, **kw):
		Task.Task.__init__(self, *k, **kw)

	def run(self): 5
		"runs a program that creates cpp files, capture the output to compile them"
		node = self.inputs[0]

		dir = self.generator.bld.srcnode.bldpath(self.env)
		cmd = 'cd %s && %s %s' % (dir, self.env['SHPIP_COMPILER'], node.abspath(self.env))
		try:
			# read the contents of the file and create cpp files from it
			files = os.popen(cmd).read().strip()
		except:
			# comment the following line to disable debugging
			#raise
			return 1 # error

		# the variable lst should contain a list of paths to the files produced
		lst = Utils.to_list(files)

		# Waf does not know "magically" what files are produced
		# In the most general case it may be necessary to run os.listdir() to see them
		# In this demo the command outputs is giving us this list

		# the files exist in the build dir only so we do not use find_or_declare
		build_nodes = [node.parent.exclusive_build_node(x) for x in lst]
		self.outputs = build_nodes

		# create the cpp tasks, in the thread
		self.more_tasks = self.add_cpp_tasks(build_nodes) 6

		# cache the file names and the task signature
		node = self.inputs[0]
		sig = self.signature()
		self.generator.bld.raw_deps[self.unique_id()] = [sig] + lst

		return 0 # no error

	def runnable_status(self):
		# look at the cache, if the shpip task was already run
		# and if the source has not changed, create the corresponding cpp tasks

		for t in self.run_after:
			if not t.hasrun:
				return ASK_LATER

		tree = self.generator.bld
		node = self.inputs[0]
		try: 7
			sig = self.signature()
			key = self.unique_id()
			deps = tree.raw_deps[key]
			prev_sig = tree.task_sigs[key][0]
		except KeyError:
			pass
		else:
			# if the file has not changed, create the cpp tasks
			if prev_sig == sig:
				lst = [self.task_gen.path.exclusive_build_node(y) for y in deps[1:]]
				self.set_outputs(lst)
				lst = self.add_cpp_tasks(lst) 8
				for tsk in lst:
					generator = self.generator.bld.generator
					generator.outstanding.append(tsk)


		if not self.outputs:
			return RUN_ME

		# this is a part of Task.Task:runnable_status: first node does not exist -> run
		# this is necessary after a clean
		env = self.env
		node = self.outputs[0]
		variant = node.variant(env)

		try:
			time = tree.node_sigs[variant][node.id]
		except KeyError:
			debug("run task #%d - the first node does not exist" % self.idx, 'task')
			try: new_sig = self.signature()
			except KeyError:
				return RUN_ME

			ret = self.can_retrieve_cache(new_sig)
			return ret and SKIP_ME or RUN_ME

		return SKIP_ME

	def add_cpp_tasks(self, lst): 9
		"creates cpp tasks after the build has started"
		tgen = self.task_gen
		tsklst = []

		for node in lst:
			TaskGen.task_gen.mapped['c_hook'](tgen, node)
			task = tgen.compiled_tasks[-1]
			task.set_run_after(self)

			# important, no link before compilations are all over
			try:
				self.generator.link_task.set_run_after(task)
			except AttributeError:
				pass

			tgen.link_task.inputs.append(task.outputs[0])
			tsklst.append(task)

			# if headers are produced something like this can be done
			# to add the include paths
			dir = task.inputs[0].parent
			# include paths for c++ and c
			self.env.append_unique('_CXXINCFLAGS', '-I%s' % dir.abspath(self.env))
			self.env.append_unique('_CCINCFLAGS', '-I%s' % dir.abspath(self.env))
			self.env.append_value('INC_PATHS', dir) # for the waf preprocessor

		return tsklst

			

1

An example. The source line contains a directive foo.shpip which triggers the creation of a shpip task (it does not represent a real file)

2

This method is used to create the shpip task when a file ending in .shpip is found

3

Create the new task type

4

Disable the warnings raised because the task has no input and outputs

5

Execute the task

7

Retrieve the information on the source files created

9

Create the c++ tasks used for processing the source files found

6

If the tasks are created during a task execution (in an execution thread), the tasks must be re-injected by adding them to the attribute more_tasks

8

If the tasks are created during the task examination (runnable_status), the tasks can be injected directly in the build by using the attribute outstanding of the scheduler