Call lv2specgen as a separate process

This speeds up builds with documentation significantly by allowing
lv2specgen.py to be called in parallel, and makes lv2specgen a more
reusable tool for third-party specifications.

3 files changed, 163 insertions, 167 deletions

diff --git a/lv2/lv2plug.in/ns/ext/dynmanifest/dynmanifest.ttl b/lv2/lv2plug.in/ns/ext/dynmanifest/dynmanifest.ttl
index 71980a1..63e0730 100644
--- a/lv2/lv2plug.in/ns/ext/dynmanifest/dynmanifest.ttl
+++ b/lv2/lv2plug.in/ns/ext/dynmanifest/dynmanifest.ttl
@@ -7,7 +7,8 @@
 
 <http://lv2plug.in/ns/ext/dynmanifest>
 	a lv2:Specification ;
-	rdfs:seeAlso <dynmanifest.h> ;
+	rdfs:seeAlso <dynmanifest.h> ,
+		<lv2-dynmanifest.doap.ttl> ;
     lv2:documentation """
 <p>The LV2 API, on its own, cannot be used to write plugin libraries where data
 is dynamically generated at runtime (e.g. API wrappers), since LV2 requires
diff --git a/lv2specgen/lv2specgen.py b/lv2specgen/lv2specgen.py
index d3b6606..62413be 100755
--- a/lv2specgen/lv2specgen.py
+++ b/lv2specgen/lv2specgen.py
@@ -41,6 +41,7 @@ __license__ = "MIT License <http://www.opensource.org/licenses/mit>"
 __contact__ = "devel@lists.lv2plug.in"
 
 import datetime
+import glob
 import optparse
 import os
 import re
@@ -1107,7 +1108,92 @@ def load_tags(path, docdir):
 
     return linkmap
 
-def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root_link=None):
+
+def writeIndex(model, index_path):
+    # Get extension URI
+    ext_node = model.value(None, rdf.type, lv2.Specification)
+    if not ext_node:
+        print('no extension found in %s' % bundle)
+        sys.exit(1)
+
+    ext = str(ext_node)
+
+    # Get version
+    minor = 0
+    micro = 0
+    try:
+        minor = int(model.value(ext_node, lv2.minorVersion, None))
+        micro = int(model.value(ext_node, lv2.microVersion, None))
+    except:
+        e = sys.exc_info()[1]
+        print('warning: %s: failed to find version for %s' % (bundle, ext))
+
+    # Get date
+    date = None
+    for r in model.triples([ext_node, doap.release, None]):
+        revision = model.value(r[2], doap.revision, None)
+        if str(revision) == ('%d.%d' % (minor, micro)):
+            date = model.value(r[2], doap.created, None)
+            break
+
+    # Verify that this date is the latest
+    for r in model.triples([ext_node, doap.release, None]):
+        this_date = model.value(r[2], doap.created, None)
+        if this_date > date:
+            print('warning: %s revision %d.%d (%s) is not the latest release' % (
+                ext_node, minor, micro, date))
+            break
+
+    # Get name and short description
+    name      = model.value(ext_node, doap.name, None)
+    shortdesc = model.value(ext_node, doap.shortdesc, None)
+
+    # Chop 'LV2' prefix from name for cleaner index
+    if name.startswith('LV2 '):
+        name = name[4:]
+
+    # Specification (comment is to act as a sort key)
+    target = os.path.relpath(path, root_path)
+    if not options.online_docs:
+        target += '/%s.html' % b
+    row = '<tr><!-- %s --><td><a rel="rdfs:seeAlso" href="%s">%s</a></td>' % (
+        b, target, name)
+
+    # API
+    row += '<td><a rel="rdfs:seeAlso" href="../doc/html/group__%s.html">%s</a></td>' % (
+        b, b)
+
+    # Description
+    if shortdesc:
+        row += '<td>' + str(shortdesc) + '</td>'
+    else:
+        row += '<td></td>'
+
+    # Version
+    version_str = '%s.%s' % (minor, micro)
+    if minor == 0 or (micro % 2 != 0):
+        row += '<td><span style="color: red">' + version_str + '</span></td>'
+    else:
+        row += '<td>' + version_str + '</td>'
+
+    # Status
+    deprecated = model.value(ext_node, owl.deprecated, None)
+    if minor == 0:
+        row += '<td><span class="error">Experimental</span></td>'
+    elif deprecated and str(deprecated[2]) != "false":
+        row += '<td><span class="warning">Deprecated</span></td>'
+    elif micro % 2 == 0:
+        row += '<td><span class="success">Stable</span></td>'
+
+    row += '</tr>'
+
+    # index = open(os.path.join(out, 'index_rows', b), 'w')
+    index = open(index_path, 'w')
+    index.write(row)
+    index.close()
+
+
+def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root_link=None, index_path=None):
     """The meat and potatoes: Everything starts here."""
 
     global spec_url
@@ -1279,6 +1365,9 @@ def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root
     template = template.replace('@DATE@', datetime.datetime.utcnow().strftime('%F'))
     template = template.replace('@TIME@', datetime.datetime.utcnow().strftime('%F %H:%M UTC'))
 
+    if index_path is not None:
+        writeIndex(m, index_path)
+
     return template
 
 
@@ -1320,7 +1409,7 @@ def getOntologyNS(m):
 
 def usage():
     script = os.path.basename(sys.argv[0])
-    return """Usage: %s ONTOLOGY INDIR STYLE OUTPUT [DOCDIR TAGS] [FLAGS]
+    return """Usage: %s ONTOLOGY INDIR STYLE OUTPUT [INDEX_PATH DOCDIR TAGS] [FLAGS]
 
         ONTOLOGY  : Path to ontology file
         INDIR     : Input directory containing template.html and style.css
@@ -1329,10 +1418,6 @@ def usage():
         DOCDIR    : Doxygen HTML directory
         TAGS      : Doxygen tags file
 
-        Optional flags:
-                -i        : Document class/property instances (disabled by default)
-                -p PREFIX : Set ontology namespace prefix from command line
-
 Example:
     %s lv2_foos.ttl template.html style.css lv2_foos.html ../doc -i -p foos
 """ % (script, script)
@@ -1342,10 +1427,17 @@ if __name__ == "__main__":
 
     opt = optparse.OptionParser(usage=usage(),
                                 description='Write HTML documentation for an RDF ontology.')
-    opt.add_option('--list-email', type='string', dest='list_email')
-    opt.add_option('--list-page', type='string', dest='list_page')
-    opt.add_option('-i', action='store_true', dest='instances')
-    opt.add_option('-p', type='string', dest='prefix')
+    opt.add_option('--list-email', type='string', dest='list_email',
+                   help='Mailing list email address')
+    opt.add_option('--list-page', type='string', dest='list_page',
+                   help='Mailing list info page address')
+    opt.add_option('-r', '--root', type='string', dest='root', default='', help='Root path')
+    opt.add_option('-p', '--prefix', type='string', dest='prefix',
+                   help='Specification Turtle prefix')
+    opt.add_option('-i', '--instances', action='store_true', dest='instances',
+                   help='Document instances')
+    opt.add_option('--online', action='store_true', dest='online_docs',
+                   help='Generate online documentation')
 
     (options, args) = opt.parse_args()
     opts = vars(options)
@@ -1354,15 +1446,48 @@ if __name__ == "__main__":
         print(usage())
         sys.exit(-1)
 
-    spec_pre = options.prefix
-    ontology = "file:" + str(args[0])
-    indir    = args[1]
-    style    = args[2]
-    output   = args[3]
-    docdir   = None
-    tags     = None
+    spec_pre   = options.prefix
+    ontology   = "file:" + str(args[0])
+    indir      = args[1]
+    style      = args[2]
+    output     = args[3]
+    index_path = None
+    docdir     = None
+    tags       = None
     if len(args) > 5:
-        docdir = args[4]
-        tags   = args[5]
+        index_path = args[4]
+        docdir     = args[5]
+        tags       = args[6]
+
+    out    = '.'
+    spec   = args[0]
+    path   = os.path.dirname(spec)
+    outdir = os.path.abspath(os.path.join(out, path))
+
+    bundle = str(outdir)
+    b = os.path.basename(outdir)
+
+    if not os.access(os.path.abspath(spec), os.R_OK):
+        print('warning: extension %s has no %s.ttl file' % (b, b))
+        sys.exit(1)
 
-    save(output, specgen(ontology, indir, style, docdir, tags, opts, instances=options.instances))
+    # Root link
+    root_path = opts['root']
+    root_link = os.path.relpath(root_path, path)
+    if not options.online_docs:
+        root_link = os.path.join(root_link, 'index.html')
+
+    # Generate spec documentation
+    specdoc = specgen(
+        os.path.abspath(spec),
+        indir,
+        style,
+        docdir,
+        tags,
+        opts,
+        instances=True,
+        root_link=root_link,
+        index_path=index_path)
+
+    # Save to HTML output file
+    save(output, specdoc)
diff --git a/wscript b/wscript
index 5f23d88..91aee03 100644
--- a/wscript
+++ b/wscript
@@ -100,140 +100,6 @@ def chop_lv2_prefix(s):
         return s[len('lv2/lv2plug.in/'):]
     return s
 
-# Rule for calling lv2specgen on a spec bundle
-def specgen(task):
-    import rdflib
-    doap = rdflib.Namespace('http://usefulinc.com/ns/doap#')
-    lv2  = rdflib.Namespace('http://lv2plug.in/ns/lv2core#')
-    owl  = rdflib.Namespace('http://www.w3.org/2002/07/owl#')
-    rdf  = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#')
-
-    sys.path.append('./lv2specgen')
-    import lv2specgen
-
-    spec   = task.inputs[0]
-    path   = os.path.dirname(spec.srcpath())
-    outdir = os.path.abspath(os.path.join(out, chop_lv2_prefix(path)))
-
-    bundle = str(outdir)
-    b = os.path.basename(outdir)
-
-    if not os.access(spec.abspath(), os.R_OK):
-        print('warning: extension %s has no %s.ttl file' % (b, b))
-        return
-
-    try:
-        model = load_ttl(glob.glob('%s/*.ttl' % bundle))
-    except:
-        e = sys.exc_info()[1]
-        print('error parsing %s: %s' % (bundle, str(e)))
-        return
-
-    # Get extension URI
-    ext_node = model.value(None, rdf.type, lv2.Specification)
-    if not ext_node:
-        print('no extension found in %s' % bundle)
-        return
-
-    ext = str(ext_node)
-
-    # Get version
-    minor = 0
-    micro = 0
-    try:
-        minor = int(model.value(ext_node, lv2.minorVersion, None))
-        micro = int(model.value(ext_node, lv2.microVersion, None))
-    except:
-        e = sys.exc_info()[1]
-        print('warning: %s: failed to find version for %s' % (bundle, ext))
-
-    # Get date
-    date = None
-    for r in model.triples([ext_node, doap.release, None]):
-        revision = model.value(r[2], doap.revision, None)
-        if str(revision) == ('%d.%d' % (minor, micro)):
-            date = model.value(r[2], doap.created, None)
-            break
-
-    # Verify that this date is the latest
-    for r in model.triples([ext_node, doap.release, None]):
-        this_date = model.value(r[2], doap.created, None)
-        if this_date > date:
-            print('warning: %s revision %d.%d (%s) is not the latest release' % (
-                ext_node, minor, micro, date))
-            break
-
-    # Get name and short description
-    name      = model.value(ext_node, doap.name, None)
-    shortdesc = model.value(ext_node, doap.shortdesc, None)
-
-    # Chop 'LV2' prefix from name for cleaner index
-    if name.startswith('LV2 '):
-        name = name[4:]
-
-    # Root link
-    ctx = task.generator.bld
-    root_path = path[0:path.find('ns/') + 3]
-    root_link = os.path.relpath(root_path, path)
-    if not task.generator.bld.env.ONLINE_DOCS:
-        root_link = os.path.join(root_link, 'index.html')
-
-    SPECGENDIR = 'lv2specgen'
-    STYLEPATH  = 'build/aux/style.css'
-    TAGFILE    = 'build/doc/tags'
-
-    specdoc = lv2specgen.specgen(
-        spec.abspath(),
-        SPECGENDIR,
-        os.path.relpath(STYLEPATH, bundle),
-        os.path.relpath('build/doc/html', bundle),
-        TAGFILE,
-        { 'list_email': 'devel@lists.lv2plug.in',
-          'list_page': 'http://lists.lv2plug.in/listinfo.cgi/devel-lv2plug.in' },
-        instances=True,
-        root_link=root_link)
-
-    lv2specgen.save(task.outputs[0].abspath(), specdoc)
-
-    # Specification (comment is to act as a sort key)
-    target = path[len('lv2/lv2plug.in/ns/'):]
-    if not task.env.ONLINE_DOCS:
-        target += '/%s.html' % b
-    row = '<tr><!-- %s --><td><a rel="rdfs:seeAlso" href="%s">%s</a></td>' % (
-        b, target, name)
-
-    # API
-    row += '<td><a rel="rdfs:seeAlso" href="../doc/html/group__%s.html">%s</a></td>' % (
-        b, b)
-
-    # Description
-    if shortdesc:
-        row += '<td>' + str(shortdesc) + '</td>'
-    else:
-        row += '<td></td>'
-
-    # Version
-    version_str = '%s.%s' % (minor, micro)
-    if minor == 0 or (micro % 2 != 0):
-        row += '<td><span style="color: red">' + version_str + '</span></td>'
-    else:
-        row += '<td>' + version_str + '</td>'
-
-    # Status
-    deprecated = model.value(ext_node, owl.deprecated, None)
-    if minor == 0:
-        row += '<td><span class="error">Experimental</span></td>'
-    elif deprecated and str(deprecated[2]) != "false":
-        row += '<td><span class="warning">Deprecated</span></td>'
-    elif micro % 2 == 0:
-        row += '<td><span class="success">Stable</span></td>'
-
-    row += '</tr>'
-
-    index = open(os.path.join('build', 'index_rows', b), 'w')
-    index.write(row)
-    index.close()
-
 def subst_file(template, output, dict):
     i = open(template, 'r')
     o = open(output, 'w')
@@ -469,23 +335,26 @@ def build(bld):
 
         # Build Doxygen documentation (and tags file)
         autowaf.build_dox(bld, 'LV2', VERSION, top, out, 'lv2plug.in/doc', False)
+        bld.add_group()
 
         index_files = []
-
-        # Call lv2specgen for each spec
         for i in specs:
-            name = os.path.basename(i.srcpath())
-            index_file = os.path.join('index_rows', name)
-            index_files += [index_file]
-
-            bld.add_group()  # Barrier (don't call lv2specgen in parallel)
-
             # Call lv2specgen to generate spec docs
-            bld(rule   = specgen,
-                name   = 'specgen',
+            name         = os.path.basename(i.srcpath())
+            index_file   = os.path.join('index_rows', name)
+            index_files += [index_file]
+            root_path    = os.path.relpath('lv2/lv2plug.in/ns', name)
+            html_path    = '%s/%s.html' % (chop_lv2_prefix(i.srcpath()), name)
+            out_bundle   = os.path.dirname(html_path)
+            bld(rule = '../lv2specgen/lv2specgen.py --root=' + root_path +
+                ' --list-email=devel@lists.lv2plug.in'
+                ' --list-page=http://lists.lv2plug.in/listinfo.cgi/devel-lv2plug.in'
+                ' ${SRC} ../lv2specgen ' +
+                os.path.relpath('aux/style.css', out_bundle) +
+                ' ${TGT} %s doc/tags' %
+                os.path.relpath('doc/html', os.path.dirname(html_path)),
                 source = os.path.join(i.srcpath(), name + '.ttl'),
-                target = ['%s/%s.html' % (chop_lv2_prefix(i.srcpath()), name),
-                          index_file])
+                target = [html_path, index_file])
 
             # Install documentation
             if not bld.env.ONLINE_DOCS:
@@ -494,6 +363,7 @@ def build(bld):
                                   bld.path.get_bld().ant_glob(base + '/*.html'))
 
         index_files.sort()
+        bld.add_group()
 
         # Build extension index
         bld(rule   = build_index,