diff options
Diffstat (limited to 'lv2specgen/lv2specgen.py')
| -rwxr-xr-x | lv2specgen/lv2specgen.py | 1840 |
1 files changed, 984 insertions, 856 deletions
diff --git a/lv2specgen/lv2specgen.py b/lv2specgen/lv2specgen.py index 62413be..c8112de 100755 --- a/lv2specgen/lv2specgen.py +++ b/lv2specgen/lv2specgen.py @@ -1,70 +1,73 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # -# lv2specgen, a documentation generator for LV2 specifications. -# Copyright (c) 2009-2014 David Robillard <d@drobilla.net> +# Copyright 2009-2026 David Robillard <d@drobilla.net> +# Copyright 2003-2008 Christopher Schmidt <crschmidt@crschmidt.net> +# Copyright 2005-2008 Uldis Bojars <uldis.bojars@deri.org> +# Copyright 2007-2008 Sergio Fernández <sergio.fernandez@fundacionctic.org> +# SPDX-License-Identifier: MIT # # Based on SpecGen: # <http://forge.morfeo-project.org/wiki_en/index.php/SpecGen> -# Copyright (c) 2003-2008 Christopher Schmidt <crschmidt@crschmidt.net> -# Copyright (c) 2005-2008 Uldis Bojars <uldis.bojars@deri.org> -# Copyright (c) 2007-2008 Sergio Fernández <sergio.fernandez@fundacionctic.org> -# -# This software is licensed under the terms of the MIT License. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -__date__ = "2011-10-26" -__version__ = __date__.replace('-', '.') -__authors__ = """ -Christopher Schmidt, -Uldis Bojars, -Sergio Fernández, -David Robillard""" -__license__ = "MIT License <http://www.opensource.org/licenses/mit>" -__contact__ = "devel@lists.lv2plug.in" + +"""Ontology specification generator tool.""" + +# pylint: disable=broad-exception-caught +# pylint: disable=c-extension-no-member +# pylint: disable=cell-var-from-loop +# pylint: disable=consider-using-f-string +# pylint: disable=global-statement +# pylint: disable=import-outside-toplevel +# pylint: disable=invalid-name +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=no-member +# pylint: disable=too-few-public-methods +# pylint: disable=too-many-arguments +# pylint: disable=too-many-branches +# pylint: disable=too-many-lines +# pylint: disable=too-many-locals +# pylint: disable=too-many-positional-arguments +# pylint: disable=too-many-statements import datetime -import glob import optparse import os import re import sys -import xml.sax.saxutils +import time import xml.dom import xml.dom.minidom +import xml.sax.saxutils + +import markdown +import markdown.extensions + +__date__ = "2026-02-07" +__version__ = __date__.replace("-", ".") +__authors__ = """ +Christopher Schmidt, +Uldis Bojars, +Sergio Fernández, +David Robillard""" +__license__ = "MIT License <http://www.opensource.org/licenses/mit>" +__contact__ = "devel@lists.lv2plug.in" try: from lxml import etree + have_lxml = True -except: +except Exception: have_lxml = False try: import pygments import pygments.lexers + import pygments.lexers.rdf import pygments.formatters - from pygments.lexer import RegexLexer, include, bygroups - from pygments.token import Text, Comment, Operator, Keyword, Name, String, Literal, Punctuation + have_pygments = True except ImportError: - print("Error importing pygments, syntax highlighting disabled") + sys.stderr.write("warning: Pygments syntax highlighting unavailable\n") have_pygments = False try: @@ -80,31 +83,42 @@ spec_url = None spec_ns_str = None spec_ns = None spec_pre = None +spec_bundle = None specgendir = None ns_list = { - "http://www.w3.org/1999/02/22-rdf-syntax-ns#" : "rdf", - "http://www.w3.org/2000/01/rdf-schema#" : "rdfs", - "http://www.w3.org/2002/07/owl#" : "owl", - "http://www.w3.org/2001/XMLSchema#" : "xsd", - "http://rdfs.org/sioc/ns#" : "sioc", - "http://xmlns.com/foaf/0.1/" : "foaf", - "http://purl.org/dc/elements/1.1/" : "dc", - "http://purl.org/dc/terms/" : "dct", - "http://purl.org/rss/1.0/modules/content/" : "content", - "http://www.w3.org/2003/01/geo/wgs84_pos#" : "geo", - "http://www.w3.org/2004/02/skos/core#" : "skos", - "http://lv2plug.in/ns/lv2core#" : "lv2", - "http://usefulinc.com/ns/doap#" : "doap", - "http://ontologi.es/doap-changeset#" : "dcs" - } - -rdf = rdflib.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#') -rdfs = rdflib.Namespace('http://www.w3.org/2000/01/rdf-schema#') -owl = rdflib.Namespace('http://www.w3.org/2002/07/owl#') -lv2 = rdflib.Namespace('http://lv2plug.in/ns/lv2core#') -doap = rdflib.Namespace('http://usefulinc.com/ns/doap#') -dcs = rdflib.Namespace('http://ontologi.es/doap-changeset#') -foaf = rdflib.Namespace('http://xmlns.com/foaf/0.1/') + "http://purl.org/dc/terms/": "dcterms", + "http://usefulinc.com/ns/doap#": "doap", + "http://xmlns.com/foaf/0.1/": "foaf", + "http://www.w3.org/2002/07/owl#": "owl", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf", + "http://www.w3.org/2000/01/rdf-schema#": "rdfs", + "http://www.w3.org/2001/XMLSchema#": "xsd", +} + +rdf = rdflib.Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#") +rdfs = rdflib.Namespace("http://www.w3.org/2000/01/rdf-schema#") +owl = rdflib.Namespace("http://www.w3.org/2002/07/owl#") +lv2 = rdflib.Namespace("http://lv2plug.in/ns/lv2core#") +doap = rdflib.Namespace("http://usefulinc.com/ns/doap#") +foaf = rdflib.Namespace("http://xmlns.com/foaf/0.1/") + + +class DTDResolver(etree.Resolver): + def resolve(self, url, _pubid, context): + path = os.path.join(specgendir, url) + return self.resolve_filename(path, context) + + +def log_error(msg): + sys.stderr.write("error: ") + sys.stderr.write(msg) + sys.stderr.write("\n") + + +def log_warning(msg): + sys.stderr.write("warning: ") + sys.stderr.write(msg) + sys.stderr.write("\n") def findStatements(model, s, p, o): @@ -112,10 +126,10 @@ def findStatements(model, s, p, o): def findOne(m, s, p, o): - l = findStatements(m, s, p, o) + triples = findStatements(m, s, p, o) try: - return l.next() - except: + return sorted(triples)[0] + except Exception: return None @@ -136,195 +150,169 @@ def getLiteralString(s): def isResource(n): - return type(n) == rdflib.URIRef + return isinstance(n, rdflib.URIRef) def isBlank(n): - return type(n) == rdflib.BNode + return isinstance(n, rdflib.BNode) def isLiteral(n): - return type(n) == rdflib.Literal + return isinstance(n, rdflib.Literal) def niceName(uri): + if uri.startswith(spec_ns_str): + return uri.replace(spec_ns_str, "") + + if uri == str(rdfs.seeAlso): + return "See also" + regexp = re.compile("^(.*[/#])([^/#]+)$") rez = regexp.search(uri) if not rez: return uri + pref = rez.group(1) if pref in ns_list: return ns_list.get(pref, pref) + ":" + rez.group(2) - else: - print("warning: prefix %s not in ns list:" % pref) - print(ns_list) - return uri + return uri -def termName(m, urinode): - "Trims the namespace out of a term to give a name to the term." + +def termName(urinode): + """Trims the namespace out of a term to give a name to the term.""" return str(urinode).replace(spec_ns_str, "") def getLabel(m, urinode): - l = findOne(m, urinode, rdfs.label, None) - if l: - return getLiteralString(getObject(l)) - else: - return '' - -if have_pygments: - # Based on sw.py by Philip Cooper - class Notation3Lexer(RegexLexer): - """ - Lexer for N3 / Turtle / NT - """ - name = 'N3' - aliases = ['n3', 'turtle'] - filenames = ['*.n3', '*.ttl', '*.nt'] - mimetypes = ['text/rdf+n3','application/x-turtle','application/n3'] - - tokens = { - 'comments': [ - (r'(\s*#.*)', Comment) - ], - 'root': [ - include('comments'), - (r'(\s*@(?:prefix|base|keywords)\s*)(\w*:\s+)?(<[^> ]*>\s*\.\s*)',bygroups(Keyword,Name.Variable,Name.Namespace)), - (r'\s*(<[^>]*\>)', Name.Class, ('triple','predObj')), - (r'(\s*[a-zA-Z_:][a-zA-Z0-9\-_:]*\s)', Name.Class, ('triple','predObj')), - (r'\s*\[\]\s*', Name.Class, ('triple','predObj')), - ], - 'triple' : [ - (r'\s*\.\s*', Text, '#pop') - ], - 'predObj': [ - include('comments'), - (r'\s*a\s*', Name.Keyword, 'object'), - (r'\s*[a-zA-Z_:][a-zA-Z0-9\-_:]*\b\s*', Name.Tag, 'object'), - (r'\s*(<[^>]*\>)', Name.Tag, 'object'), - (r'\s*\]\s*', Text, '#pop'), - (r'(?=\s*\.\s*)', Keyword, '#pop'), - ], - 'objList': [ - include('comments'), - (r'\s*\)', Text, '#pop'), - include('object') - ], - 'object': [ - include('comments'), - (r'\s*\[', Text, 'predObj'), - (r'\s*<[^> ]*>', Name.Tag), - (r'\s*("""(?:.|\n)*?""")(\@[a-z]{2-4}|\^\^<?[a-zA-Z0-9\-\:_#/\.]*>?)?\s*', bygroups(Literal.String,Text)), - (r'\s*".*?[^\\]"(?:\@[a-z]{2-4}|\^\^<?[a-zA-Z0-9\-\:_#/\.]*>?)?\s*', Literal.String), - (r'\s*[0-9]+\.[0-9]*\s*\n?', Literal.Number), - (r'\s*[0-9]+\s*\n?', Literal.Number), - (r'\s*[a-zA-Z0-9\-_\:]+\s*', Name.Tag), - (r'\s*\(', Text, 'objList'), - (r'\s*;\s*\n?', Punctuation, '#pop'), - (r'\s*,\s*\n?', Punctuation), # Added by drobilla so "," is not an error - (r'(?=\s*\])', Text, '#pop'), - (r'(?=\s*\.)', Text, '#pop'), - ], - } - -def linkify(string): - if linkmap == {}: + statement = findOne(m, urinode, rdfs.label, None) + if statement: + return getLiteralString(getObject(statement)) + + return "" + + +def linkifyCodeIdentifiers(string): + """Add links to code documentation for identifiers like LV2_Type.""" + + if not linkmap: return string - "Add links to code documentation for identifiers" - if string in linkmap.keys(): + if string in linkmap: # Exact match for complete string return linkmap[string] - rgx = re.compile('([^a-zA-Z0-9_:])(' + \ - '|'.join(map(re.escape, linkmap)) + \ - ')([^a-zA-Z0-9_:])') + rgx = re.compile( + "([^a-zA-Z0-9_:])(" + + "|".join(map(re.escape, linkmap)) + + ")([^a-zA-Z0-9_:])" + ) def translateCodeLink(match): return match.group(1) + linkmap[match.group(2)] + match.group(3) return rgx.sub(translateCodeLink, string) -def getComment(m, urinode, classlist, proplist, instalist): - c = findOne(m, urinode, lv2.documentation, None) - if c: - markup = getLiteralString(getObject(c)) - - # Syntax highlight all C code - if have_pygments: - code_rgx = re.compile('<pre class="c-code">(.*?)</pre>', re.DOTALL) - while True: - code = code_rgx.search(markup) - if not code: - break - match_str = xml.sax.saxutils.unescape(code.group(1)) - code_str = pygments.highlight( - match_str, - pygments.lexers.CLexer(), - pygments.formatters.HtmlFormatter()) - markup = code_rgx.sub(code_str, markup, 1) - - # Syntax highlight all Turtle code - if have_pygments: - code_rgx = re.compile('<pre class="turtle-code">(.*?)</pre>', re.DOTALL) - while True: - code = code_rgx.search(markup) - if not code: - break - match_str = xml.sax.saxutils.unescape(code.group(1)) - code_str = pygments.highlight( - match_str, - Notation3Lexer(), - pygments.formatters.HtmlFormatter()) - markup = code_rgx.sub(code_str, markup, 1) - - # Add links to code documentation for identifiers - markup = linkify(markup) - - # Transform prefixed names like eg:something into links if possible - rgx = re.compile('([a-zA-Z0-9_-]+):([a-zA-Z0-9_-]+)') - namespaces = getNamespaces(m) - def translateLink(match): - text = match.group(0) - prefix = match.group(1) - name = match.group(2) - curie = match.group(0) - uri = rdflib.URIRef(spec_ns + name) - if prefix == spec_pre: - if not ((classlist and uri in classlist) or - (instalist and uri in instalist) or - (proplist and uri in proplist)): - print("warning: Link to undefined resource <%s>\n" % text) - return '<a href="#%s">%s</a>' % (name, curie) - elif prefix in namespaces: - return '<a href="%s">%s</a>' % ( - namespaces[match.group(1)] + match.group(2), - match.group(0)) - else: - return text - markup = rgx.sub(translateLink, markup) - - # Transform names like #foo into links into this spec if possible - rgx = re.compile('([ \t\n\r\f\v^]+)\#([a-zA-Z0-9_-]+)') - def translateLocalLink(match): - text = match.group(0) - space = match.group(1) - name = match.group(2) - uri = rdflib.URIRef(spec_ns + name) - if ((classlist and uri in classlist) or - (instalist and uri in instalist) or - (proplist and uri in proplist)): - return '%s<a href="#%s">%s</a>' % (space, name, name) - else: - print("warning: Link to undefined resource <%s>\n" % name) - return text - markup = rgx.sub(translateLocalLink, markup) - - if have_lxml: - try: - # Parse and validate documentation as XHTML Basic 1.1 - doc = """<?xml version="1.0" encoding="UTF-8"?> + +def linkifyVocabIdentifiers(m, string, classlist, proplist, instalist): + """Add links to vocabulary documentation for prefixed names.""" + + rgx = re.compile("([a-zA-Z0-9_-]+):([a-zA-Z0-9_-]+)") + namespaces = getNamespaces(m) + + def translateLink(match): + text = match.group(0) + prefix = match.group(1) + name = match.group(2) + uri = rdflib.URIRef(spec_ns + name) + if prefix == spec_pre: + if not ( + (classlist and uri in classlist) + or (instalist and uri in instalist) + or (proplist and uri in proplist) + ): + log_warning("Link to undefined resource <%s>" % text) + return '<a href="#%s">%s</a>' % (name, name) + + if prefix in namespaces: + return '<a href="%s">%s</a>' % ( + namespaces[match.group(1)] + match.group(2), + match.group(0), + ) + + return text + + return rgx.sub(translateLink, string) + + +def prettifyHtml(m, markup, subject, classlist, proplist, instalist): + # Syntax highlight all C code + if have_pygments: + code_re = re.compile('<pre class="c-code">(.*?)</pre>', re.DOTALL) + while True: + code = code_re.search(markup) + if not code: + break + match_str = xml.sax.saxutils.unescape(code.group(1)) + code_str = pygments.highlight( + match_str, + pygments.lexers.CLexer(), + pygments.formatters.HtmlFormatter(), + ) + markup = code_re.sub(code_str, markup, 1) + + # Syntax highlight all Turtle code + if have_pygments: + code_re = re.compile('<pre class="turtle-code">(.*?)</pre>', re.DOTALL) + while True: + code = code_re.search(markup) + if not code: + break + match_str = xml.sax.saxutils.unescape(code.group(1)) + code_str = pygments.highlight( + match_str, + pygments.lexers.rdf.TurtleLexer(), + pygments.formatters.HtmlFormatter(), + ) + markup = code_re.sub(code_str, markup, 1) + + # Add links to code documentation for identifiers + markup = linkifyCodeIdentifiers(markup) + + # Add internal links for known prefixed names + markup = linkifyVocabIdentifiers(m, markup, classlist, proplist, instalist) + + # Transform names like #foo into links into this spec if possible + rgx = re.compile("([ \t\n\r\f\v^]+)#([a-zA-Z0-9_-]+)") + + def translateLocalLink(match): + text = match.group(0) + space = match.group(1) + name = match.group(2) + uri = rdflib.URIRef(spec_ns + name) + known = ( + (classlist and uri in classlist) + or (instalist and uri in instalist) + or (proplist and uri in proplist) + ) + if known: + return '%s<a href="#%s">%s</a>' % (space, name, name) + + log_warning("Link to undefined resource <%s>" % name) + return text + + markup = rgx.sub(translateLocalLink, markup) + + if not have_lxml: + log_warning("No Python lxml module found, output may be invalid") + else: + oldcwd = os.getcwd() + + try: + # Parse and validate documentation as XHTML Basic 1.1 + doc = ( + """<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML Basic 1.1//EN" "DTD/xhtml-basic11.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> @@ -332,102 +320,168 @@ def getComment(m, urinode, classlist, proplist, instalist): <title>Validation Skeleton Document</title> </head> <body> -%s +""" + + markup + + """ </body> -</html> -""" % str(markup.decode()) - - oldcwd = os.getcwd() - os.chdir(specgendir) - parser = etree.XMLParser(dtd_validation=True, no_network=True) - root = etree.fromstring(doc, parser) - os.chdir(oldcwd) - except Exception as e: - print("Invalid lv2:documentation for %s\n%s" % (urinode, e)) - line_num = 1 - for line in doc.split('\n'): - print('%3d: %s' % (line_num, line)) - line_num += 1 - - return markup - - c = findOne(m, urinode, rdfs.comment, None) +</html>""" + ) + + os.chdir(specgendir) + parser = etree.XMLParser(dtd_validation=True, no_network=True) + etree.fromstring(doc.encode("utf-8"), parser) + except Exception as e: + log_error("Invalid documentation for %s:" % subject) + sys.stderr.write(str(e) + "\n") + line_num = 1 + for line in doc.split("\n"): + sys.stderr.write("%3d: %s\n" % (line_num, line)) + line_num += 1 + finally: + os.chdir(oldcwd) + + return markup + + +def formatDoc(m, urinode, literal, classlist, proplist, instalist): + string = getLiteralString(literal) + + if literal.datatype == lv2.Markdown: + ext = [ + "markdown.extensions.codehilite", + "markdown.extensions.tables", + "markdown.extensions.def_list", + ] + + doc = markdown.markdown(string, extensions=ext) + + # Hack to make tables valid XHTML Basic 1.1 + for tag in ["thead", "tbody"]: + doc = doc.replace("<%s>\n" % tag, "") + doc = doc.replace("</%s>\n" % tag, "") + + return prettifyHtml(m, doc, urinode, classlist, proplist, instalist) + + doc = xml.sax.saxutils.escape(string) + doc = linkifyCodeIdentifiers(doc) + doc = linkifyVocabIdentifiers(m, doc, classlist, proplist, instalist) + return "<p>%s</p>" % doc + + +def getComment(m, subject, classlist, proplist, instalist): + c = findOne(m, subject, rdfs.comment, None) if c: - text = getLiteralString(getObject(c)) - return '<p>%s</p>' % xml.sax.saxutils.escape(text) + comment = getObject(c) + return formatDoc(m, subject, comment, classlist, proplist, instalist) + + return "" + + +def getDetailedDocumentation(m, subject, classlist, proplist, instalist): + markup = "" + + d = findOne(m, subject, lv2.documentation, None) + if d: + doc = getObject(d) + if doc.datatype == lv2.Markdown: + markup += formatDoc( + m, subject, doc, classlist, proplist, instalist + ) + else: + html = getLiteralString(doc) + markup += prettifyHtml( + m, html, subject, classlist, proplist, instalist + ) + + return markup - return '' + +def getFullDocumentation(m, subject, classlist, proplist, instalist): + # Use rdfs:comment for first summary line + markup = getComment(m, subject, classlist, proplist, instalist) + + # Use lv2:documentation for further details + markup += getDetailedDocumentation( + m, subject, classlist, proplist, instalist + ) + + return markup def getProperty(val, first=True): - "Return a string representing a property value in a property table" - doc = '' + """Return a string representing a property value in a property table.""" + doc = "" if not first: - doc += '<tr><td></td>' # Empty cell in header column - doc += '<td>%s</td></tr>\n' % val + doc += "<tr><th></th>" # Empty cell in header column + doc += "<td>%s</td></tr>\n" % val return doc def endProperties(first): - if first: - return '</tr>' - else: - return '' + return "</tr>" if first else "" def rdfsPropertyInfo(term, m): - """Generate HTML for properties: Domain, range""" - global classranges - global classdomains + """Generate HTML for properties: Domain, range.""" doc = "" - range = "" - domain = "" + + label = getLabel(m, term) + if label != "": + doc += "<tr><th>Label</th><td>%s</td></tr>" % label # Find subPropertyOf information - rlist = '' + rlist = "" first = True for st in findStatements(m, term, rdfs.subPropertyOf, None): k = getTermLink(getObject(st), term, rdfs.subPropertyOf) rlist += getProperty(k, first) first = False - if rlist != '': - doc += '<tr><th>Sub-property of</th>' + rlist + if rlist != "": + doc += "<tr><th>Sub-property of</th>" + rlist # Domain stuff domains = findStatements(m, term, rdfs.domain, None) domainsdoc = "" first = True - for d in domains: + for d in sorted(domains): union = findOne(m, getObject(d), owl.unionOf, None) if union: uris = parseCollection(m, getObject(union)) for uri in uris: - domainsdoc += getProperty(getTermLink(uri, term, rdfs.domain), first) + domainsdoc += getProperty( + getTermLink(uri, term, rdfs.domain), first + ) add(classdomains, uri, term) else: if not isBlank(getObject(d)): - domainsdoc += getProperty(getTermLink(getObject(d), term, rdfs.domain), first) + domainsdoc += getProperty( + getTermLink(getObject(d), term, rdfs.domain), first + ) first = False - if (len(domainsdoc) > 0): + if len(domainsdoc) > 0: doc += "<tr><th>Domain</th>%s" % domainsdoc # Range stuff ranges = findStatements(m, term, rdfs.range, None) rangesdoc = "" first = True - for r in ranges: + for r in sorted(ranges): union = findOne(m, getObject(r), owl.unionOf, None) if union: uris = parseCollection(m, getObject(union)) for uri in uris: - rangesdoc += getProperty(getTermLink(uri, term, rdfs.range), first) + rangesdoc += getProperty( + getTermLink(uri, term, rdfs.range), first + ) add(classranges, uri, term) first = False else: if not isBlank(getObject(r)): - rangesdoc += getProperty(getTermLink(getObject(r), term, rdfs.range), first) + rangesdoc += getProperty( + getTermLink(getObject(r), term, rdfs.range), first + ) first = False - if (len(rangesdoc) > 0): + if len(rangesdoc) > 0: doc += "<tr><th>Range</th>%s" % rangesdoc return doc @@ -438,9 +492,9 @@ def parseCollection(model, node): while node: first = findOne(model, node, rdf.first, None) - rest = findOne(model, node, rdf.rest, None) + rest = findOne(model, node, rdf.rest, None) if not first or not rest: - break; + break uris.append(getObject(first)) node = getObject(rest) @@ -450,41 +504,37 @@ def parseCollection(model, node): def getTermLink(uri, subject=None, predicate=None): uri = str(uri) - extra = '' - if subject != None and predicate != None: - extra = 'about="%s" rel="%s" resource="%s"' % (str(subject), niceName(str(predicate)), uri) - if (uri.startswith(spec_ns_str)): - return '<a href="#%s" %s>%s</a>' % (uri.replace(spec_ns_str, ""), extra, niceName(uri)) - else: - return '<a href="%s" %s>%s</a>' % (uri, extra, niceName(uri)) + extra = "" + if subject is not None and predicate is not None: + extra = 'about="%s" rel="%s" resource="%s"' % ( + str(subject), + niceName(str(predicate)), + uri, + ) + if uri.startswith(spec_ns_str): + return '<a href="#%s" %s>%s</a>' % ( + uri.replace(spec_ns_str, ""), + extra, + niceName(uri), + ) + + return '<a href="%s" %s>%s</a>' % (uri, extra, niceName(uri)) + + +def owlRestrictionInfo(term, m): + """Generate OWL restriction information for Classes.""" - -def rdfsClassInfo(term, m): - """Generate rdfs-type information for Classes: ranges, and domains.""" - global classranges - global classdomains - doc = "" - - # Find subClassOf information restrictions = [] - superclasses = [] - for st in findStatements(m, term, rdfs.subClassOf, None): - if not isBlank(getObject(st)): - uri = getObject(st) - if not uri in superclasses: - superclasses.append(uri) - else: - meta_type = findOne(m, getObject(st), rdf.type, None) - restrictions.append(getSubject(meta_type)) + for s in findStatements(m, term, rdfs.subClassOf, None): + if findOne(m, getObject(s), rdf.type, owl.Restriction): + restrictions.append(getObject(s)) - if len(superclasses) > 0: - doc += "\n<tr><th>Sub-class of</th>" - first = True - for superclass in superclasses: - doc += getProperty(getTermLink(superclass), first) - first = False + if not restrictions: + return "" + + doc = "<dl>" - for r in restrictions: + for r in sorted(restrictions): props = findStatements(m, r, None, None) onProp = None comment = None @@ -493,46 +543,80 @@ def rdfsClassInfo(term, m): onProp = getObject(p) elif getPredicate(p) == rdfs.comment: comment = getObject(p) - if onProp != None: - doc += '<tr><th>Restriction on %s</th><td>' % getTermLink(onProp) + if onProp is not None: + doc += "<dt>Restriction on %s</dt>\n" % getTermLink(onProp) - prop_str = '' - last_pred = None - first = True + prop_str = "" for p in findStatements(m, r, None, None): - if (getPredicate(p) == owl.onProperty + if ( + getPredicate(p) == owl.onProperty or getPredicate(p) == rdfs.comment - or (getPredicate(p) == rdf.type and getObject(p) == owl.Restriction) - or getPredicate(p) == lv2.documentation): - last_pred = None + or ( + getPredicate(p) == rdf.type + and getObject(p) == owl.Restriction + ) + or getPredicate(p) == lv2.documentation + ): continue - if getPredicate(p) != last_pred: - prop_str += '<tr><th>%s</th>\n' % getTermLink(getPredicate(p)) - first = True + prop_str += getTermLink(getPredicate(p)) + if isResource(getObject(p)): - prop_str += getProperty(getTermLink(getObject(p)), first) - first = False + prop_str += " " + getTermLink(getObject(p)) elif isLiteral(getObject(p)): - prop_str += getProperty(getLiteralString(getObject(p)), first) - first = False + prop_str += " " + getLiteralString(getObject(p)) - last_pred = getPredicate(p) + if comment is not None: + prop_str += "\n<div>%s</div>\n" % getLiteralString(comment) - prop_str += endProperties(first) + doc += "<dd>%s</dd>" % prop_str if prop_str else "" - if prop_str != '': - doc += '<table class=\"restriction\">%s</table>\n' % prop_str - if comment != None: - doc += "<span>%s</span>\n" % getLiteralString(comment) - doc += '</td></tr>' + doc += "</dl>" + return doc + + +def rdfsClassInfo(term, m): + """Generate rdfs-type information for Classes: ranges, and domains.""" + doc = "" + + label = getLabel(m, term) + if label != "": + doc += "<tr><th>Label</th><td>%s</td></tr>" % label + + # Find superclasses + superclasses = set() + for st in findStatements(m, term, rdfs.subClassOf, None): + if not isBlank(getObject(st)): + uri = getObject(st) + superclasses |= set([uri]) + + if len(superclasses) > 0: + doc += "\n<tr><th>Subclass of</th>" + first = True + for superclass in sorted(superclasses): + doc += getProperty(getTermLink(superclass), first) + first = False + + # Find subclasses + subclasses = set() + for st in findStatements(m, None, rdfs.subClassOf, term): + if not isBlank(getObject(st)): + uri = getSubject(st) + subclasses |= set([uri]) + + if len(subclasses) > 0: + doc += "\n<tr><th>Superclass of</th>" + first = True + for superclass in sorted(subclasses): + doc += getProperty(getTermLink(superclass), first) + first = False # Find out about properties which have rdfs:domain of t d = classdomains.get(str(term), "") if d: - dlist = '' + dlist = "" first = True - for k in d: + for k in sorted(d): dlist += getProperty(getTermLink(k), first) first = False doc += "<tr><th>In domain of</th>%s" % dlist @@ -540,9 +624,9 @@ def rdfsClassInfo(term, m): # Find out about properties which have rdfs:range of t r = classranges.get(str(term), "") if r: - rlist = '' + rlist = "" first = True - for k in r: + for k in sorted(r): rlist += getProperty(getTermLink(k), first) first = False doc += "<tr><th>In range of</th>%s" % rlist @@ -551,71 +635,95 @@ def rdfsClassInfo(term, m): def isSpecial(pred): - """Return True if the predicate is "special" and shouldn't be emitted generically""" - return pred in [rdf.type, rdfs.range, rdfs.domain, rdfs.label, rdfs.comment, rdfs.subClassOf, rdfs.subPropertyOf, lv2.documentation] + """Return True if `pred` shouldn't be documented generically.""" + + return pred in [ + rdf.type, + rdfs.range, + rdfs.domain, + rdfs.label, + rdfs.comment, + rdfs.subClassOf, + rdfs.subPropertyOf, + lv2.documentation, + owl.withRestrictions, + ] def blankNodeDesc(node, m): properties = findStatements(m, node, None, None) - doc = '' - last_pred = '' + doc = "" for p in sorted(properties): if isSpecial(getPredicate(p)): continue - doc += '<tr>' + doc += "<tr>" doc += '<td class="blankterm">%s</td>\n' % getTermLink(getPredicate(p)) if isResource(getObject(p)): doc += '<td class="blankdef">%s</td>\n' % getTermLink(getObject(p)) # getTermLink(str(getObject(p)), node, getPredicate(p)) elif isLiteral(getObject(p)): - doc += '<td class="blankdef">%s</td>\n' % getLiteralString(getObject(p)) + doc += '<td class="blankdef">%s</td>\n' % getLiteralString( + getObject(p) + ) elif isBlank(getObject(p)): - doc += '<td class="blankdef">' + blankNodeDesc(getObject(p), m) + '</td>\n' + doc += ( + '<td class="blankdef">' + + blankNodeDesc(getObject(p), m) + + "</td>\n" + ) else: doc += '<td class="blankdef">?</td>\n' - doc += '</tr>' - if doc != '': + doc += "</tr>" + if doc != "": doc = '<table class="blankdesc">\n%s\n</table>\n' % doc return doc def extraInfo(term, m): - """Generate information about misc. properties of a term""" + """Generate information about misc. properties of a term.""" doc = "" properties = findStatements(m, term, None, None) first = True for p in sorted(properties): if isSpecial(getPredicate(p)): continue - doc += '<tr><th>%s</th>\n' % getTermLink(getPredicate(p)) + doc += "<tr><th>%s</th>\n" % getTermLink(getPredicate(p)) if isResource(getObject(p)): - doc += getProperty(getTermLink(getObject(p), term, getPredicate(p)), first) + doc += getProperty( + getTermLink(getObject(p), term, getPredicate(p)), first + ) elif isLiteral(getObject(p)): - doc += getProperty(linkify(str(getObject(p))), first) + doc += getProperty( + linkifyCodeIdentifiers(str(getObject(p))), first + ) elif isBlank(getObject(p)): doc += getProperty(str(blankNodeDesc(getObject(p), m)), first) else: - doc += getProperty('?', first) + doc += getProperty("?", first) - #doc += endProperties(first) + # doc += endProperties(first) return doc def rdfsInstanceInfo(term, m): - """Generate rdfs-type information for instances""" + """Generate rdfs-type information for instances.""" doc = "" + label = getLabel(m, term) + if label != "": + doc += "<tr><th>Label</th><td>%s</td></tr>" % label + first = True - for match in findStatements(m, term, rdf.type, None): - doc += getProperty(getTermLink(getObject(match), - term, - rdf.type), - first) + types = "" + for match in sorted(findStatements(m, term, rdf.type, None)): + types += getProperty( + getTermLink(getObject(match), term, rdf.type), first + ) first = False - if doc != "": - doc = "<tr><th>Type</th>" + doc + if types != "": + doc += "<tr><th>Type</th>" + types doc += endProperties(first) @@ -624,7 +732,7 @@ def rdfsInstanceInfo(term, m): def owlInfo(term, m): """Returns an extra information that is defined about a term using OWL.""" - res = '' + res = "" # Inverse properties ( owl:inverseOf ) first = True @@ -637,85 +745,84 @@ def owlInfo(term, m): def owlTypeInfo(term, propertyType, name): if findOne(m, term, rdf.type, propertyType): - return "<tr><th>OWL Type</th><td>%s</td></tr>\n" % name - else: - return "" + return "<tr><th>Type</th><td>%s</td></tr>\n" % name + + return "" res += owlTypeInfo(term, owl.DatatypeProperty, "Datatype Property") res += owlTypeInfo(term, owl.ObjectProperty, "Object Property") res += owlTypeInfo(term, owl.AnnotationProperty, "Annotation Property") - res += owlTypeInfo(term, owl.InverseFunctionalProperty, "Inverse Functional Property") + res += owlTypeInfo( + term, owl.InverseFunctionalProperty, "Inverse Functional Property" + ) res += owlTypeInfo(term, owl.SymmetricProperty, "Symmetric Property") return res + def isDeprecated(m, subject): deprecated = findOne(m, subject, owl.deprecated, None) return deprecated and (str(deprecated[2]).find("true") >= 0) -def docTerms(category, list, m, classlist, proplist, instalist): + +def docTerms(category, termlist, m, classlist, proplist, instalist): """ A wrapper class for listing all the terms in a specific class (either Properties, or Classes. Category is 'Property' or 'Class', list is a - list of term names (strings), return value is a chunk of HTML. + list of term URI strings, return value is a chunk of HTML. """ doc = "" - nspre = spec_pre - for item in list: - t = termName(m, item) - if (t.startswith(spec_ns_str)) and ( - len(t[len(spec_ns_str):].split("/")) < 2): - term = t - t = t.split(spec_ns_str[-1])[1] - curie = "%s:%s" % (nspre, t) - else: - if t.startswith("http://"): - term = t - curie = getShortName(t) - t = getAnchor(t) - else: - term = spec_ns[t] - curie = "%s:%s" % (nspre, t) + for term in termlist: + if not term.startswith(spec_ns_str): + continue - term_uri = term + t = termName(term) + curie = term.split(spec_ns_str[-1])[1] + if t: + doc += '<div class="specterm" id="%s" about="%s">' % (t, term) + else: + doc += '<div class="specterm" about="%s">' % term - doc += """<div class="specterm" id="%s" about="%s">\n<h3>%s <a href="#%s">%s</a></h3>\n""" % (t, term_uri, category, getAnchor(str(term_uri)), curie) + doc += '<h3><a href="#%s">%s</a></h3>' % (getAnchor(term), curie) + doc += '<span class="spectermtype">%s</span>' % category - label = getLabel(m, term) - comment = getComment(m, term, classlist, proplist, instalist) + comment = getFullDocumentation(m, term, classlist, proplist, instalist) is_deprecated = isDeprecated(m, term) doc += '<div class="spectermbody">' - if label != '' or comment != '' or is_deprecated: - doc += '<div class="description">' - - if label != '': - doc += "<div property=\"rdfs:label\" class=\"label\">%s</div>" % label - - if is_deprecated: - doc += '<div class="warning">DEPRECATED</div>' - - if comment != '': - doc += "<div property=\"rdfs:comment\">%s</div>" % comment - - if label != '' or comment != '' or is_deprecated: - doc += "</div>" terminfo = "" - if category == 'Property': - terminfo += owlInfo(term, m) + extrainfo = "" + if category == "Property": terminfo += rdfsPropertyInfo(term, m) - if category == 'Class': + terminfo += owlInfo(term, m) + if category == "Class": terminfo += rdfsClassInfo(term, m) - if category == 'Instance': + extrainfo += owlRestrictionInfo(term, m) + if category == "Instance": terminfo += rdfsInstanceInfo(term, m) terminfo += extraInfo(term, m) - if (len(terminfo) > 0): # to prevent empty list (bug #882) + if len(terminfo) > 0: # to prevent empty list (bug #882) doc += '\n<table class="terminfo">%s</table>\n' % terminfo - doc += '</div>' + doc += '<div class="description">' + + if is_deprecated: + doc += '<div class="warning">Deprecated</div>' + + if comment != "": + doc += ( + '<div class="comment" property="rdfs:comment">%s</div>' + % comment + ) + + doc += extrainfo + + doc += "</div>" + + doc += "</div>" doc += "\n</div>\n\n" return doc @@ -723,40 +830,39 @@ def docTerms(category, list, m, classlist, proplist, instalist): def getShortName(uri): uri = str(uri) - if ("#" in uri): - return uri.split("#")[-1] - else: - return uri.split("/")[-1] + if "#" in uri: + return uri.split("#", maxsplit=1)[-1] + + return uri.split("/", maxsplit=1)[-1] def getAnchor(uri): uri = str(uri) - if (uri.startswith(spec_ns_str)): - return uri[len(spec_ns_str):].replace("/", "_") - else: - return getShortName(uri) + if uri.startswith(spec_ns_str): + return uri.replace(spec_ns_str, "").replace("/", "_") + + return getShortName(uri) -def buildIndex(m, classlist, proplist, instalist=None, filelist=None): - if not (classlist or proplist or instalist or filelist): - return '' +def buildIndex(m, classlist, proplist, instalist=None): + if not (classlist or proplist or instalist): + return "" - head = '<tr>' - body = '<tr>' + head = "" + body = "" - def termLink(m, t): + def termLink(t): if str(t).startswith(spec_ns_str): - name = termName(m, t) + name = termName(t) return '<a href="#%s">%s</a>' % (name, name) - else: - return '<a href="%s">%s</a>' % (str(t), str(t)) - if (len(classlist) > 0): - head += '<th>Classes</th>' - body += '<td><ul>' - classlist.sort() + return '<a href="%s">%s</a>' % (str(t), str(t)) + + if len(classlist) > 0: + head += '<th><a href="#ref-classes" />Classes</th>' + body += "<td><ul>" shown = {} - for c in classlist: + for c in sorted(classlist): if c in shown: continue @@ -764,104 +870,111 @@ def buildIndex(m, classlist, proplist, instalist=None, filelist=None): local_subclass = False for p in findStatements(m, c, rdfs.subClassOf, None): parent = str(p[2]) - if parent[0:len(spec_ns_str)] == spec_ns_str: + if parent.startswith(spec_ns_str): local_subclass = True if local_subclass: continue shown[c] = True - body += '<li>' + termLink(m, c) + body += "<li>" + termLink(c) + def class_tree(c): - tree = '' + tree = "" shown[c] = True subclasses = [] for s in findStatements(m, None, rdfs.subClassOf, c): subclasses += [getSubject(s)] - subclasses.sort() - for s in subclasses: - tree += '<li>' + termLink(m, s) + for s in sorted(subclasses): + tree += "<li>" + termLink(s) tree += class_tree(s) - tree += '</li>' - if tree != '': - tree = '<ul>' + tree + '</ul>' + tree += "</li>" + if tree != "": + tree = "<ul>" + tree + "</ul>" return tree + body += class_tree(c) - body += '</li>' - body += '</ul></td>\n' - - if (len(proplist) > 0): - head += '<th>Properties</th>' - body += '<td><ul>' - proplist.sort() - for p in proplist: - body += '<li>%s</li>' % termLink(m, p) - body += '</ul></td>\n' - - if (instalist != None and len(instalist) > 0): - head += '<th>Instances</th>' - body += '<td><ul>' - instalist.sort() - for i in instalist: + body += "</li>" + body += "</ul></td>\n" + + if len(proplist) > 0: + head += '<th><a href="#ref-properties" />Properties</th>' + body += "<td><ul>" + for p in sorted(proplist): + body += "<li>%s</li>" % termLink(p) + body += "</ul></td>\n" + + if instalist is not None and len(instalist) > 0: + head += '<th><a href="#ref-instances" />Instances</th>' + body += "<td><ul>" + for i in sorted(instalist): p = getShortName(i) anchor = getAnchor(i) body += '<li><a href="#%s">%s</a></li>' % (anchor, p) - body += '</ul></td>\n' + body += "</ul></td>\n" - if (filelist != None and len(filelist) > 0): - head += '<th>Files</th>' - body += '<td><ul>' - filelist.sort() - for i in filelist: - p = getShortName(i) - anchor = getAnchor(i) - body += '<li><a href="%s">%s</a></li>' % (i, os.path.basename(i)) - body += '</ul></td>\n' + if head and body: + return """<table class="index"> +<thead><tr>%s</tr></thead> +<tbody><tr>%s</tr></tbody></table> +""" % ( + head, + body, + ) - head += '</tr>' - body += '</tr>' - return '<table class="index"><thead>%s</thead>\n<tbody>%s</tbody></table>' % (head, body) + return "" def add(where, key, value): - if not key in where: + if key not in where: where[key] = [] - if not value in where[key]: + if value not in where[key]: where[key].append(value) def specInformation(m, ns): + """Read through the spec model and return classlist and proplist. + + Global variables classranges and classdomains are also filled as + appropriate. """ - Read through the spec (provided as a Redland model) and return classlist - and proplist. Global variables classranges and classdomains are also filled - as appropriate. - """ - global classranges - global classdomains # Find the class information: Ranges, domains, and list of all names. classtypes = [rdfs.Class, owl.Class, rdfs.Datatype] classlist = [] for onetype in classtypes: for classStatement in findStatements(m, None, rdf.type, onetype): - for range in findStatements(m, None, rdfs.range, getSubject(classStatement)): + for rangelink in findStatements( + m, None, rdfs.range, getSubject(classStatement) + ): if not isBlank(getSubject(classStatement)): - add(classranges, + add( + classranges, str(getSubject(classStatement)), - str(getSubject(range))) - for domain in findStatements(m, None, rdfs.domain, getSubject(classStatement)): + str(getSubject(rangelink)), + ) + for domain in findStatements( + m, None, rdfs.domain, getSubject(classStatement) + ): if not isBlank(getSubject(classStatement)): - add(classdomains, + add( + classdomains, str(getSubject(classStatement)), - str(getSubject(domain))) + str(getSubject(domain)), + ) if not isBlank(getSubject(classStatement)): klass = getSubject(classStatement) if klass not in classlist and str(klass).startswith(ns): classlist.append(klass) # Create a list of properties in the schema. - proptypes = [rdf.Property, owl.ObjectProperty, owl.DatatypeProperty, owl.AnnotationProperty] + proptypes = [ + rdf.Property, + owl.ObjectProperty, + owl.DatatypeProperty, + owl.AnnotationProperty, + ] proplist = [] for onetype in proptypes: for propertyStatement in findStatements(m, None, rdf.type, onetype): @@ -873,14 +986,14 @@ def specInformation(m, ns): def specProperty(m, subject, predicate): - "Return a property of the spec." + """Return a property of the spec.""" for c in findStatements(m, subject, predicate, None): return getLiteralString(getObject(c)) - return '' + return "" def specProperties(m, subject, predicate): - "Return a property of the spec." + """Return a property of the spec.""" properties = [] for c in findStatements(m, subject, predicate, None): properties += [getObject(c)] @@ -888,9 +1001,9 @@ def specProperties(m, subject, predicate): def specAuthors(m, subject): - "Return an HTML description of the authors of the spec." + """Return an HTML description of the authors of the spec.""" - subjects = [subject]; + subjects = [subject] p = findOne(m, subject, lv2.project, None) if p: subjects += [getObject(p)] @@ -907,110 +1020,44 @@ def specAuthors(m, subject): for j in findStatements(m, getObject(i), foaf.name, None): maint.add(getLiteralString(getObject(j))) - doc = '' + doc = "" - devdoc = '' + devdoc = "" first = True - for d in dev: + for d in sorted(dev): if not first: - devdoc += ', ' - devdoc += '<span class="author" property="doap:developer">%s</span>' % d + devdoc += ", " + + devdoc += f'<span class="author" property="doap:developer">{d}</span>' first = False if len(dev) == 1: - doc += '<tr><th class="metahead">Developer</th><td>%s</td></tr>' % devdoc + doc += f'<tr><th class="metahead">Developer</th><td>{devdoc}</td></tr>' elif len(dev) > 0: - doc += '<tr><th class="metahead">Developers</th><td>%s</td></tr>' % devdoc + doc += ( + f'<tr><th class="metahead">Developers</th><td>{devdoc}</td></tr>' + ) - maintdoc = '' + maintdoc = "" first = True - for m in maint: + for name in sorted(maint): if not first: - maintdoc += ', ' - maintdoc += '<span class="author" property="doap:maintainer">%s</span>' % m - first = False - if len(maint) == 1: - doc += '<tr><th class="metahead">Maintainer</th><td>%s</td></tr>' % maintdoc - elif len(maint) > 0: - doc += '<tr><th class="metahead">Maintainers</th><td>%s</td></tr>' % maintdoc + maintdoc += ", " - return doc - - -def releaseChangeset(m, release, prefix=''): - changeset = findOne(m, release, dcs.changeset, None) - if changeset is None: - return '' - - entry = '' - #entry = '<dd><ul>\n' - for i in findStatements(m, getObject(changeset), dcs.item, None): - item = getObject(i) - label = findOne(m, item, rdfs.label, None) - if not label: - print("error: dcs:item has no rdfs:label") - continue - - text = getLiteralString(getObject(label)) - if prefix: - text = prefix + ': ' + text - - entry += '<li>%s</li>\n' % text - - #entry += '</ul></dd>\n' - return entry - - -def specHistoryEntries(m, subject, entries): - for r in findStatements(m, subject, doap.release, None): - release = getObject(r) - revNode = findOne(m, release, doap.revision, None) - if not revNode: - print("error: doap:release has no doap:revision") - continue - - rev = getLiteralString(getObject(revNode)) - - created = findOne(m, release, doap.created, None) - - dist = findOne(m, release, doap['file-release'], None) - if dist: - entry = '<dt><a href="%s">Version %s</a>' % (getObject(dist), rev) - else: - entry = '<dt>Version %s' % rev - #print("warning: doap:release has no doap:file-release") - - if created: - entry += ' (%s)</dt>\n' % getLiteralString(getObject(created)) - else: - entry += ' (<span class="warning">EXPERIMENTAL</span>)</dt>' - - entry += '<dd><ul>\n%s' % releaseChangeset(m, release) - - if dist is not None: - entries[(getObject(created), getObject(dist))] = entry - - return entries - - -def specHistoryMarkup(entries): - if len(entries) > 0: - history = '<dl>\n' - for e in sorted(entries.keys(), reverse=True): - history += entries[e] + '</ul></dd>' - history += '</dl>\n' - return history - else: - return '' + maintdoc += '<span class="author" property="doap:maintainer">' + maintdoc += name + maintdoc += "</span>" + first = False + if maint: + label = "Maintainer" if len(maint) == 1 else "Maintainers" + doc += f'<tr><th class="metahead">{label}</th><td>{maintdoc}</td></tr>' -def specHistory(m, subject): - return specHistoryMarkup(specHistoryEntries(m, subject, {})) + return doc def specVersion(m, subject): - """ - Return a (minorVersion, microVersion, date) tuple - """ + """Return a (minorVersion, microVersion, date) tuple.""" + # Get the date from the latest doap release latest_doap_revision = "" latest_doap_release = None @@ -1021,7 +1068,7 @@ def specVersion(m, subject): latest_doap_revision = revision latest_doap_release = getObject(i) date = "" - if latest_doap_release != None: + if latest_doap_release is not None: for i in findStatements(m, latest_doap_release, doap.created, None): date = getLiteralString(getObject(i)) @@ -1036,10 +1083,7 @@ def specVersion(m, subject): def getInstances(model, classes, properties): - """ - Extract all resources instanced in the ontology - (aka "everything that is not a class or a property") - """ + """Extract all non-class and non-property instances in the ontology.""" instances = [] for c in classes: for i in findStatements(model, None, rdf.type, c): @@ -1049,18 +1093,21 @@ def getInstances(model, classes, properties): if inst not in instances and str(inst) != spec_url: instances.append(inst) for i in findStatements(model, None, rdf.type, None): - if ((not isResource(getSubject(i))) + if ( + (not isResource(getSubject(i))) or (getSubject(i) in classes) or (getSubject(i) in instances) - or (getSubject(i) in properties)): + or (getSubject(i) in properties) + ): continue full_uri = str(getSubject(i)) - if (full_uri.startswith(spec_ns_str)): + if full_uri.startswith(spec_ns_str): instances.append(getSubject(i)) return instances + def load_tags(path, docdir): - "Build a (symbol => URI) map from a Doxygen tag file." + """Build a (symbol => URI) map from a Doxygen tag file.""" if not path or not docdir: return {} @@ -1068,162 +1115,106 @@ def load_tags(path, docdir): def getChildText(elt, tagname): "Return the content of the first child node with a certain tag name." for e in elt.childNodes: - if e.nodeType == xml.dom.Node.ELEMENT_NODE and e.tagName == tagname: + if ( + e.nodeType == xml.dom.Node.ELEMENT_NODE + and e.tagName == tagname + ): return e.firstChild.nodeValue - return '' + return "" def linkTo(filename, anchor, sym): if anchor: - return '<span><a href="%s/%s#%s">%s</a></span>' % (docdir, filename, anchor, sym) - else: - return '<span><a href="%s/%s">%s</a></span>' % (docdir, filename, sym) - - tagdoc = xml.dom.minidom.parse(path) - root = tagdoc.documentElement - linkmap = {} + return '<span><a href="%s/%s#%s">%s</a></span>' % ( + docdir, + filename, + anchor, + sym, + ) + + return '<span><a href="%s/%s">%s</a></span>' % ( + docdir, + filename, + sym, + ) + + tagdoc = xml.dom.minidom.parse(path) + root = tagdoc.documentElement + result = {} for cn in root.childNodes: - if (cn.nodeType == xml.dom.Node.ELEMENT_NODE - and cn.tagName == 'compound' - and cn.getAttribute('kind') != 'page'): - - name = getChildText(cn, 'name') - filename = getChildText(cn, 'filename') - anchor = getChildText(cn, 'anchor') - if not filename.endswith('.html'): - filename += '.html' - - if cn.getAttribute('kind') != 'group': - linkmap[name] = linkTo(filename, anchor, name) - - prefix = '' - if cn.getAttribute('kind') == 'struct': - prefix = name + '::' - - members = cn.getElementsByTagName('member') + if ( + cn.nodeType == xml.dom.Node.ELEMENT_NODE + and cn.tagName == "compound" + and cn.getAttribute("kind") != "page" + ): + name = getChildText(cn, "name") + filename = getChildText(cn, "filename") + anchor = getChildText(cn, "anchor") + if not filename.endswith(".html"): + filename += ".html" + + if cn.getAttribute("kind") != "group": + result[name] = linkTo(filename, anchor, name) + + prefix = "" + if cn.getAttribute("kind") == "struct": + prefix = name + "::" + + members = cn.getElementsByTagName("member") for m in members: - mname = prefix + getChildText(m, 'name') - mafile = getChildText(m, 'anchorfile') - manchor = getChildText(m, 'anchor') - linkmap[mname] = linkTo(mafile, manchor, mname) - - return linkmap - - -def writeIndex(model, index_path): - # Get extension URI - ext_node = model.value(None, rdf.type, lv2.Specification) - if not ext_node: - print('no extension found in %s' % bundle) - sys.exit(1) - - ext = str(ext_node) - - # Get version - minor = 0 - micro = 0 - try: - minor = int(model.value(ext_node, lv2.minorVersion, None)) - micro = int(model.value(ext_node, lv2.microVersion, None)) - except: - e = sys.exc_info()[1] - print('warning: %s: failed to find version for %s' % (bundle, ext)) - - # Get date - date = None - for r in model.triples([ext_node, doap.release, None]): - revision = model.value(r[2], doap.revision, None) - if str(revision) == ('%d.%d' % (minor, micro)): - date = model.value(r[2], doap.created, None) - break - - # Verify that this date is the latest - for r in model.triples([ext_node, doap.release, None]): - this_date = model.value(r[2], doap.created, None) - if this_date > date: - print('warning: %s revision %d.%d (%s) is not the latest release' % ( - ext_node, minor, micro, date)) - break - - # Get name and short description - name = model.value(ext_node, doap.name, None) - shortdesc = model.value(ext_node, doap.shortdesc, None) - - # Chop 'LV2' prefix from name for cleaner index - if name.startswith('LV2 '): - name = name[4:] - - # Specification (comment is to act as a sort key) - target = os.path.relpath(path, root_path) - if not options.online_docs: - target += '/%s.html' % b - row = '<tr><!-- %s --><td><a rel="rdfs:seeAlso" href="%s">%s</a></td>' % ( - b, target, name) - - # API - row += '<td><a rel="rdfs:seeAlso" href="../doc/html/group__%s.html">%s</a></td>' % ( - b, b) - - # Description - if shortdesc: - row += '<td>' + str(shortdesc) + '</td>' - else: - row += '<td></td>' - - # Version - version_str = '%s.%s' % (minor, micro) - if minor == 0 or (micro % 2 != 0): - row += '<td><span style="color: red">' + version_str + '</span></td>' - else: - row += '<td>' + version_str + '</td>' - - # Status - deprecated = model.value(ext_node, owl.deprecated, None) - if minor == 0: - row += '<td><span class="error">Experimental</span></td>' - elif deprecated and str(deprecated[2]) != "false": - row += '<td><span class="warning">Deprecated</span></td>' - elif micro % 2 == 0: - row += '<td><span class="success">Stable</span></td>' - - row += '</tr>' - - # index = open(os.path.join(out, 'index_rows', b), 'w') - index = open(index_path, 'w') - index.write(row) - index.close() - - -def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root_link=None, index_path=None): + mname = prefix + getChildText(m, "name") + mafile = getChildText(m, "anchorfile") + manchor = getChildText(m, "anchor") + result[mname] = linkTo(mafile, manchor, mname) + + return result + + +def specgen( + specloc, + template_path, + style_uri, + docdir, + tags, + opts, + instances=False, + validate=False, +): """The meat and potatoes: Everything starts here.""" + global spec_bundle global spec_url global spec_ns_str global spec_ns global spec_pre - global ns_list - global specgendir global linkmap - specgendir = os.path.abspath(indir) + spec_bundle = "file://%s/" % os.path.abspath(os.path.dirname(specloc)) # Template - temploc = os.path.join(indir, "template.html") - template = None - f = open(temploc, "r") - template = f.read() - f.close() + with open(template_path, "r", encoding="utf-8") as f: + template = f.read() # Load code documentation link map from tags file linkmap = load_tags(tags, docdir) - m = rdflib.ConjunctiveGraph() - manifest_path = os.path.join(os.path.dirname(specloc), 'manifest.ttl') + # Create a new empty dataset + rdflib_major = int(rdflib.__version__.split(".")[0]) + rdflib_minor = int(rdflib.__version__.split(".")[1]) + if rdflib_major > 7 or rdflib_major == 7 and rdflib_minor >= 5: + m = rdflib.Dataset() + else: + m = rdflib.ConjunctiveGraph() + + # RDFLib adds its own prefixes, so kludge around "time" prefix conflict + m.namespace_manager.bind( + "time", rdflib.URIRef("http://lv2plug.in/ns/ext/time#"), replace=True + ) + + manifest_path = os.path.join(os.path.dirname(specloc), "manifest.ttl") if os.path.exists(manifest_path): - m.parse(manifest_path, format='n3') - m.parse(specloc, format='n3') + m.parse(manifest_path, format="n3") + m.parse(specloc, format="n3") - bundle_path = os.path.split(specloc[specloc.find(':') + 1:])[0] - abs_bundle_path = os.path.abspath(bundle_path) spec_url = getOntologyNS(m) spec = rdflib.URIRef(spec_url) @@ -1233,39 +1224,42 @@ def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root while not done: done = True for uri in specProperties(m, spec, rdfs.seeAlso): - if uri[:7] == 'file://': + if uri[:7] == "file://": path = uri[7:] - if (path != os.path.abspath(specloc) and - path.endswith('ttl') and - path not in seeAlso): + if ( + path != os.path.abspath(specloc) + and path.endswith("ttl") + and path not in seeAlso + ): seeAlso.add(path) - m.parse(path, format='n3') + m.parse(path, format="n3") done = False spec_ns_str = spec_url - if (spec_ns_str[-1] != "/" and spec_ns_str[-1] != "#"): + if spec_ns_str[-1] != "/" and spec_ns_str[-1] != "#": spec_ns_str += "#" spec_ns = rdflib.Namespace(spec_ns_str) namespaces = getNamespaces(m) - keys = namespaces.keys() - keys.sort() + keys = sorted(namespaces.keys()) prefixes_html = "<span>" for i in keys: uri = namespaces[i] - if uri.startswith('file:'): - continue; + if uri.startswith("file:"): + continue ns_list[str(uri)] = i - if (str(uri) == spec_url + '#' or - str(uri) == spec_url + '/' or - str(uri) == spec_url): + if ( + str(uri) == spec_url + "#" + or str(uri) == spec_url + "/" + or str(uri) == spec_url + ): spec_pre = i prefixes_html += '<a href="%s">%s</a> ' % (uri, i) prefixes_html += "</span>" if spec_pre is None: - print('No namespace prefix for %s defined' % specloc) + log_error("No namespace prefix for %s defined" % specloc) sys.exit(1) ns_list[spec_ns_str] = spec_pre @@ -1276,67 +1270,73 @@ def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root instalist = None if instances: - instalist = getInstances(m, classlist, proplist) - instalist.sort(lambda x, y: cmp(getShortName(x).lower(), getShortName(y).lower())) + instalist = sorted( + getInstances(m, classlist, proplist), + key=lambda x: getShortName(x).lower(), + ) - filelist = [] - see_also_files = specProperties(m, spec, rdfs.seeAlso) - see_also_files.sort() - for f in see_also_files: - uri = str(f) - if uri[:7] == 'file://': - uri = uri[7:] - if uri[:len(abs_bundle_path)] == abs_bundle_path: - uri = uri[len(abs_bundle_path) + 1:] - else: - continue # Skip seeAlso file outside bundle - - filelist += [uri] - - azlist = buildIndex(m, classlist, proplist, instalist, filelist) + azlist = buildIndex(m, classlist, proplist, instalist) # Generate Term HTML - termlist = docTerms('Property', proplist, m, classlist, proplist, instalist) - termlist = docTerms('Class', classlist, m, classlist, proplist, instalist) + termlist + classlist = docTerms("Class", classlist, m, classlist, proplist, instalist) + proplist = docTerms( + "Property", proplist, m, classlist, proplist, instalist + ) + instlist = "" if instances: - termlist += docTerms('Instance', instalist, m, classlist, proplist, instalist) + instlist = docTerms( + "Instance", instalist, m, classlist, proplist, instalist + ) + + termlist = "" + if classlist: + termlist += '<div class="section">' + termlist += '<h2><a id="ref-classes" />Classes</h2>' + classlist + termlist += "</div>" + + if proplist: + termlist += '<div class="section">' + termlist += '<h2><a id="ref-properties" />Properties</h2>' + proplist + termlist += "</div>" + + if instlist: + termlist += '<div class="section">' + termlist += '<h2><a id="ref-instances" />Instances</h2>' + instlist + termlist += "</div>" name = specProperty(m, spec, doap.name) title = name - if root_link: - name = '<a href="%s">%s</a>' % (root_link, name) - - template = template.replace('@TITLE@', title) - template = template.replace('@NAME@', name) - template = template.replace('@SHORT_DESC@', specProperty(m, spec, doap.shortdesc)) - template = template.replace('@URI@', spec) - template = template.replace('@PREFIX@', spec_pre) - if spec_pre == 'lv2': - template = template.replace('@XMLNS@', '') - else: - template = template.replace('@XMLNS@', ' xmlns:%s="%s"' % (spec_pre, spec_ns_str)) + + template = template.replace("@TITLE@", title) + template = template.replace("@NAME@", name) + template = template.replace( + "@SHORT_DESC@", specProperty(m, spec, doap.shortdesc) + ) + template = template.replace("@URI@", spec) + template = template.replace("@PREFIX@", spec_pre) filename = os.path.basename(specloc) - basename = filename[0:filename.rfind('.')] - - template = template.replace('@STYLE_URI@', style_uri) - template = template.replace('@PREFIXES@', str(prefixes_html)) - template = template.replace('@BASE@', spec_ns_str) - template = template.replace('@AUTHORS@', specAuthors(m, spec)) - template = template.replace('@INDEX@', azlist) - template = template.replace('@REFERENCE@', termlist) - template = template.replace('@FILENAME@', filename) - template = template.replace('@HEADER@', basename + '.h') - template = template.replace('@HISTORY@', specHistory(m, spec)) - - mail_row = '' - if 'list_email' in opts: + basename = os.path.splitext(filename)[0] + + template = template.replace("@STYLE_URI@", style_uri) + template = template.replace("@PREFIXES@", str(prefixes_html)) + template = template.replace("@BASE@", spec_ns_str) + template = template.replace("@AUTHORS@", specAuthors(m, spec)) + template = template.replace("@INDEX@", azlist) + template = template.replace("@REFERENCE@", termlist) + template = template.replace("@FILENAME@", filename) + template = template.replace("@HEADER@", basename + ".h") + + mail_row = "" + if "list_email" in opts: mail_row = '<tr><th>Discuss</th><td><a href="mailto:%s">%s</a>' % ( - opts['list_email'], opts['list_email']) - if 'list_page' in opts: - mail_row += ' <a href="%s">(subscribe)</a>' % opts['list_page'] - mail_row += '</td></tr>' - template = template.replace('@MAIL@', mail_row) + opts["list_email"], + opts["list_email"], + ) + if "list_page" in opts: + mail_row += ' <a href="%s">(subscribe)</a>' % opts["list_page"] + mail_row += "</td></tr>" + template = template.replace("@MAIL@", mail_row) version = specVersion(m, spec) # (minor, micro, date) date_string = version[2] @@ -1344,49 +1344,72 @@ def specgen(specloc, indir, style_uri, docdir, tags, opts, instances=False, root date_string = "Undated" version_string = "%s.%s" % (version[0], version[1]) - experimental = (version[0] == 0 or version[1] % 2 == 1) + experimental = version[0] == 0 or version[1] % 2 == 1 if experimental: version_string += ' <span class="warning">EXPERIMENTAL</span>' if isDeprecated(m, rdflib.URIRef(spec_url)): version_string += ' <span class="warning">DEPRECATED</span>' - template = template.replace('@VERSION@', version_string) + template = template.replace("@VERSION@", version_string) - content_links = '<li><a href="%s">API</a></li>' % os.path.join(docdir, 'group__%s.html' % basename) - template = template.replace('@CONTENT_LINKS@', content_links) + content_links = "" + if docdir is not None: + content_links = '<li><a href="%s">API</a></li>' % os.path.join( + docdir, "group__%s.html" % basename + ) - comment = getComment(m, rdflib.URIRef(spec_url), classlist, proplist, instalist) - if comment != '': - template = template.replace('@COMMENT@', comment) - else: - template = template.replace('@COMMENT@', '') + template = template.replace("@CONTENT_LINKS@", content_links) + + docs = getDetailedDocumentation( + m, rdflib.URIRef(spec_url), classlist, proplist, instalist + ) + template = template.replace("@DESCRIPTION@", docs) - template = template.replace('@DATE@', datetime.datetime.utcnow().strftime('%F')) - template = template.replace('@TIME@', datetime.datetime.utcnow().strftime('%F %H:%M UTC')) + now = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) + build_date = datetime.datetime.fromtimestamp(now, datetime.timezone.utc) + template = template.replace("@DATE@", build_date.strftime("%F")) + template = template.replace("@TIME@", build_date.strftime("%F %H:%M UTC")) + if not validate: + return template - if index_path is not None: - writeIndex(m, index_path) + # Validate complete output page + oldcwd = os.getcwd() + try: + os.chdir(specgendir) + parser = etree.XMLParser(dtd_validation=True, no_network=True) + parser.resolvers.add(DTDResolver()) + + etree.fromstring( + template.replace( + '"http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd"', + '"DTD/xhtml-rdfa-1.dtd"', + ).encode("utf-8"), + parser, + ) + except Exception as e: + log_error("Validation failed for %s: %s" % (specloc, e)) + finally: + os.chdir(oldcwd) return template def save(path, text): try: - f = open(path, "w") - f.write(text.encode("utf-8")) - f.flush() - f.close() + with open(path, "w", encoding="utf-8") as f: + f.write(text) + f.flush() except Exception: e = sys.exc_info()[1] - print('Error writing to file "' + path + '": ' + str(e)) + log_error('Error writing to file "%s": %s' % (path, e)) def getNamespaces(m): - """Return a prefix:URI dictionary of all namespaces seen during parsing""" + """Return a prefix:URI dictionary of all namespaces seen during parsing.""" nspaces = {} for prefix, uri in m.namespaces(): - if not re.match('default[0-9]*', prefix) and not prefix == 'xml': + if not re.match("default[0-9]*", prefix) and not prefix == "xml": # Skip silly default namespaces added by rdflib nspaces[prefix] = uri return nspaces @@ -1401,7 +1424,7 @@ def getOntologyNS(m): if not isBlank(getSubject(s)): ns = str(getSubject(s)) - if (ns == None): + if ns is None: sys.exit("Impossible to get ontology's namespace") else: return ns @@ -1409,85 +1432,190 @@ def getOntologyNS(m): def usage(): script = os.path.basename(sys.argv[0]) - return """Usage: %s ONTOLOGY INDIR STYLE OUTPUT [INDEX_PATH DOCDIR TAGS] [FLAGS] + return "Usage: %s ONTOLOGY_TTL OUTPUT_HTML [OPTION]..." % script - ONTOLOGY : Path to ontology file - INDIR : Input directory containing template.html and style.css - STYLE : Stylesheet URI - OUTPUT : HTML output path - DOCDIR : Doxygen HTML directory - TAGS : Doxygen tags file -Example: - %s lv2_foos.ttl template.html style.css lv2_foos.html ../doc -i -p foos -""" % (script, script) +def _path_from_env(variable, default): + value = os.environ.get(variable) + return value if value and os.path.isabs(value) else default -if __name__ == "__main__": - """Ontology specification generator tool""" - - opt = optparse.OptionParser(usage=usage(), - description='Write HTML documentation for an RDF ontology.') - opt.add_option('--list-email', type='string', dest='list_email', - help='Mailing list email address') - opt.add_option('--list-page', type='string', dest='list_page', - help='Mailing list info page address') - opt.add_option('-r', '--root', type='string', dest='root', default='', help='Root path') - opt.add_option('-p', '--prefix', type='string', dest='prefix', - help='Specification Turtle prefix') - opt.add_option('-i', '--instances', action='store_true', dest='instances', - help='Document instances') - opt.add_option('--online', action='store_true', dest='online_docs', - help='Generate online documentation') + +def _paths_from_env(variable, default): + paths = [] + value = os.environ.get(variable) + if value: + paths = [p for p in value.split(os.pathsep) if os.path.isabs(p)] + + return paths if paths else default + + +def _data_dirs(): + return _paths_from_env( + "XDG_DATA_DIRS", ["/usr/local/share/", "/usr/share/"] + ) + + +def main(): + """Main program that parses the program arguments and runs.""" + + global spec_pre + global specgendir + + script_dir = os.path.dirname(os.path.realpath(__file__)) + if os.path.exists(os.path.join(script_dir, "template.html")): + # Run from source repository + specgendir = script_dir + lv2_source_root = os.path.dirname(script_dir) + default_template_path = os.path.join(script_dir, "template.html") + default_style_dir = os.path.join(lv2_source_root, "doc", "style") + else: + data_dir = None + for d in _data_dirs(): + path = os.path.join(d, "lv2specgen") + files_exist = all( + os.path.exists(os.path.join(path, name)) + for name in ["template.html", "style.css", "pygments.css"] + ) + if files_exist: + data_dir = path + break + + if data_dir: + # Use installed files + specgendir = data_dir + default_template_path = os.path.join(data_dir, "template.html") + default_style_dir = data_dir + else: + log_error("Unable to find lv2specgen data") + sys.exit(-2) + + opt = optparse.OptionParser( + usage=usage(), + description="Write HTML documentation for an RDF ontology.", + ) + opt.add_option( + "--list-email", + type="string", + dest="list_email", + help="Mailing list email address", + ) + opt.add_option( + "--list-page", + type="string", + dest="list_page", + help="Mailing list info page address", + ) + opt.add_option( + "--template", + type="string", + dest="template", + default=default_template_path, + help="Template file for output page", + ) + opt.add_option( + "--style-dir", + type="string", + dest="style_dir", + default=default_style_dir, + help="Stylesheet directory path", + ) + opt.add_option( + "--style-uri", + type="string", + dest="style_uri", + default="style.css", + help="Stylesheet URI", + ) + opt.add_option( + "--docdir", + type="string", + dest="docdir", + default=None, + help="Doxygen output directory", + ) + opt.add_option( + "--tags", + type="string", + dest="tags", + default=None, + help="Doxygen tags file", + ) + opt.add_option( + "-p", + "--prefix", + type="string", + dest="prefix", + help="Specification Turtle prefix", + ) + opt.add_option( + "-i", + "--instances", + action="store_true", + dest="instances", + help="Document instances", + ) + opt.add_option( + "--copy-style", + action="store_true", + dest="copy_style", + help="Copy style from template directory to output directory", + ) + opt.add_option( + "--validate", + action="store_true", + dest="validate", + help="Validate against local XML DTDs", + ) (options, args) = opt.parse_args() opts = vars(options) - if (len(args) < 3): - print(usage()) + if len(args) < 2: + opt.print_help() sys.exit(-1) - spec_pre = options.prefix - ontology = "file:" + str(args[0]) - indir = args[1] - style = args[2] - output = args[3] - index_path = None - docdir = None - tags = None - if len(args) > 5: - index_path = args[4] - docdir = args[5] - tags = args[6] - - out = '.' - spec = args[0] - path = os.path.dirname(spec) + spec_pre = options.prefix + output = args[1] + docdir = options.docdir + tags = options.tags + + out = "." + spec = args[0] + path = os.path.dirname(spec) outdir = os.path.abspath(os.path.join(out, path)) - bundle = str(outdir) b = os.path.basename(outdir) if not os.access(os.path.abspath(spec), os.R_OK): - print('warning: extension %s has no %s.ttl file' % (b, b)) + log_error("extension %s has no %s.ttl file" % (b, b)) sys.exit(1) - # Root link - root_path = opts['root'] - root_link = os.path.relpath(root_path, path) - if not options.online_docs: - root_link = os.path.join(root_link, 'index.html') - # Generate spec documentation specdoc = specgen( - os.path.abspath(spec), - indir, - style, + spec, + opts["template"], + opts["style_uri"], docdir, tags, opts, instances=True, - root_link=root_link, - index_path=index_path) + validate=opts["validate"], + ) # Save to HTML output file save(output, specdoc) + + if opts["copy_style"]: + import shutil + + for stylesheet in ["pygments.css", "style.css"]: + style_dir = opts["style_dir"] + output_dir = os.path.dirname(output) + shutil.copyfile( + os.path.join(style_dir, stylesheet), + os.path.join(output_dir, stylesheet), + ) + + +if __name__ == "__main__": + main() |