QGIS/tests/code_layout/doxygen_parser.py
2024-11-29 15:38:02 +01:00

732 lines
26 KiB
Python

"""
***************************************************************************
mocked
---------------------
Date : May 2017
Copyright : (C) 2017 by Denis Rouzaud
Email : denis.rouzaud@gmail.com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""
__author__ = "Denis Rouzaud"
__date__ = "May 2017"
__copyright__ = "(C) 2017, Denis Rouzaud"
import glob
import os
import re
try:
import xml.etree.ElementTree as ET
except ImportError:
import xml.etree.ElementTree as ET
class DoxygenParser:
"""
Parses the XML files generated by Doxygen which describe the API docs
"""
def __init__(
self,
path,
acceptable_missing={},
acceptable_missing_added_note=[],
acceptable_missing_brief=[],
):
"""
Initializes the parser.
:param path: Path to Doxygen XML output
"""
self.acceptable_missing = acceptable_missing
self.acceptable_missing_added_note = acceptable_missing_added_note
self.acceptable_missing_brief = acceptable_missing_brief
self.documentable_members = 0
self.documented_members = 0
self.undocumented_members = {}
self.noncompliant_members = {}
self.broken_links = {}
self.bindable_members = []
self.groups = {}
self.classes_missing_group = []
self.classes_missing_brief = []
self.classes_missing_version_added = []
# for some reason the Doxygen generation on Travis refuses to assign these classes to groups
self.acceptable_missing_group = [
"QgsOgcUtils::LayerProperties",
"QgsSQLStatement::Node",
"QgsSQLStatement::NodeBinaryOperator",
"QgsSQLStatement::NodeColumnRef",
"QgsSQLStatement::NodeFunction",
"QgsSQLStatement::NodeInOperator",
"QgsSQLStatement::NodeList",
"QgsSQLStatement::NodeLiteral",
"QgsSQLStatement::NodeUnaryOperator",
"QgsRuleBasedLabeling::Rule",
"QgsSQLStatement::Visitor",
]
self.version_regex = re.compile(
r".*QGIS\s+(?:<ref.*?>)?[\d\.]+.*", re.MULTILINE
)
self.parseFiles(path)
def parseFiles(self, path):
"""Parses all the Doxygen XML files in a folder
:param path: Path to Doxygen XML output
"""
found = False
# find groups
for f in glob.glob(os.path.join(path, "group__*.xml")):
found = True
group, members = self.parseGroup(f)
self.groups[group] = members
assert found, "Could not find doxygen groups xml"
found = False
# parse docs
for f in glob.glob(os.path.join(path, "*.xml")):
found = True
self.parseFile(f)
assert found, "Could not find doxygen files xml"
def parseGroup(self, f):
"""Parses a single Doxygen Group XML file
:param f: XML file path
"""
name = None
members = []
# Wrap everything in a try, as sometimes Doxygen XML is malformed
try:
for event, elem in ET.iterparse(f):
if event == "end" and elem.tag == "compoundname":
name = elem.text
if event == "end" and elem.tag == "innerclass":
members.append(elem.text)
except:
pass
return name, members
def hasGroup(self, class_name):
"""Returns true if a class has been assigned to a group
:param class_name class name to test
"""
for g in self.groups:
if class_name in self.groups[g]:
return True
return False
@staticmethod
def standardize_signature(signature):
"""
Standardizes a method's signature for comparison
"""
return signature.lower().replace("* >", "*>").replace("< ", "<")
def parseFile(self, f):
"""Parses a single Doxygen XML file
:param f: XML file path
"""
documentable_members = 0
documented_members = 0
# Wrap everything in a try, as sometimes Doxygen XML is malformed
try:
for event, elem in ET.iterparse(f):
if event == "end" and elem.tag == "compounddef":
if self.elemIsPublicClass(elem):
# store documentation status
(
members,
documented,
undocumented,
noncompliant,
bindable,
has_brief_description,
found_version_added,
broken_links,
) = self.parseClassElem(elem)
documentable_members += members
documented_members += documented
class_name = elem.find("compoundname").text
acceptable_missing = self.acceptable_missing.get(class_name, [])
if (
not self.hasGroup(class_name)
and class_name not in self.acceptable_missing_group
):
self.classes_missing_group.append(class_name)
if (
class_name not in self.acceptable_missing_brief
and not has_brief_description
):
self.classes_missing_brief.append(class_name)
if (
class_name not in self.acceptable_missing_added_note
and not found_version_added
):
self.classes_missing_version_added.append(class_name)
# GEN LIST
# if len(undocumented) > 0:
# print('"%s": [%s],' % (class_name, ", ".join(['"%s"' % e.replace('"', '\\"') for e in undocumented])))
unacceptable_undocumented = undocumented - set(
acceptable_missing
)
# do a case insensitive check too
unacceptable_undocumented_insensitive = {
DoxygenParser.standardize_signature(u) for u in undocumented
} - {
DoxygenParser.standardize_signature(u)
for u in acceptable_missing
}
if len(unacceptable_undocumented_insensitive) > 0:
self.undocumented_members[class_name] = {}
self.undocumented_members[class_name][
"documented"
] = documented
self.undocumented_members[class_name]["members"] = members
self.undocumented_members[class_name][
"missing_members"
] = unacceptable_undocumented
if len(noncompliant) > 0:
self.noncompliant_members[class_name] = noncompliant
if broken_links:
self.broken_links[class_name] = broken_links
# store bindable members
if self.classElemIsBindable(elem):
for m in bindable:
self.bindable_members.append(m)
elem.clear()
except ET.ParseError as e:
# sometimes Doxygen generates malformed xml (e.g., for < and > operators)
line_num, col = e.position
with open(f) as xml_file:
for i, l in enumerate(xml_file):
if i == line_num - 1:
line = l
break
caret = "{:=>{}}".format("^", col)
print(f"ParseError in {f}\n{e}\n{line}\n{caret}")
self.documentable_members += documentable_members
self.documented_members += documented_members
def elemIsPublicClass(self, elem):
"""Tests whether an XML element corresponds to a public (or protected) class
:param elem: XML element
"""
# only looking for classes
if not elem.get("kind") == "class":
return False
# only looking for public or protected classes
return elem.get("prot") in ("public", "protected")
def classElemIsBindable(self, elem):
"""Tests whether a class should have SIP bindings
:param elem: XML element corresponding to a class
"""
try:
# check for classes with special python doc notes (probably 'not available' or renamed classes, either way
# they should be safe to ignore as obviously some consideration has been given to Python bindings)
detailed_sec = elem.find("detaileddescription")
for p in detailed_sec.iter("para"):
for s in p.iter("simplesect"):
for ps in s.iter("para"):
if ps.text and "python" in ps.text.lower():
return False
return True
except:
return True
def parseClassElem(self, e):
"""Parses an XML element corresponding to a Doxygen class
:param e: XML element
"""
documentable_members = 0
documented_members = 0
undocumented_members = set()
noncompliant_members = []
bindable_members = []
broken_links = {}
# loop through all members
for m in e.iter("memberdef"):
signature = self.memberSignature(m)
if signature is None:
continue
if self.elemIsBindableMember(m):
bindable_member = [e.find("compoundname").text, m.find("name").text]
if bindable_member not in bindable_members:
bindable_members.append(bindable_member)
if self.elemIsDocumentableMember(m):
documentable_members += 1
if self.memberIsDocumented(m):
documented_members += 1
error = self.memberDocIsNonCompliant(m)
if error:
noncompliant_members.append({m.find("name").text: error})
else:
undocumented_members.add(signature)
broken_see_also_links = self.checkForBrokenSeeAlsoLinks(m)
if broken_see_also_links:
broken_links[m.find("name").text] = broken_see_also_links
# test for brief description
d = e.find("briefdescription")
has_brief_description = False
if d is not None:
has_brief_description = True
for para in d.iter("para"):
if para.text and re.search(r"\btodo\b", para.text.lower()) is not None:
noncompliant_members.append(
{
"Brief description": "Don't add TODO comments to public doxygen documentation. Leave these as c++ code comments only."
}
)
break
# test for "added in QGIS xxx" string
d = e.find("detaileddescription")
found_version_added = False
for para in d.iter("para"):
for s in para.iter("simplesect"):
if s.get("kind") == "since":
for p in s.iter("para"):
if self.version_regex.match(ET.tostring(p).decode()):
found_version_added = True
break
for s in para.iter("xrefsect"):
if (
s.find("xreftitle") is not None
and "Deprecated" in s.find("xreftitle").text
):
# can't have both deprecated and since, so if we've found deprecated then treat it as having satisfied the "since" requirement too
found_version_added = True
break
if para.text and re.search(r"\btodo\b", para.text.lower()) is not None:
noncompliant_members.append(
{
"Detailed description": "Don't add TODO comments to public doxygen documentation. Leave these as c++ code comments only."
}
)
return (
documentable_members,
documented_members,
undocumented_members,
noncompliant_members,
bindable_members,
has_brief_description,
found_version_added,
broken_links,
)
def memberSignature(self, elem):
"""Returns the signature for a member
:param elem: XML element for a class member
"""
a = elem.find("argsstring")
try:
if a is not None:
signature = elem.find("name").text + a.text
else:
signature = elem.find("name").text
if signature.endswith("= default"):
signature = signature[: -len("= default")]
return signature.strip()
except:
return None
def elemIsBindableMember(self, elem):
"""Tests whether an member should be included in SIP bindings
:param elem: XML element for a class member
"""
# only public or protected members are bindable
if not self.visibility(elem) in ("public", "protected"):
return False
# property themselves are not bound, only getters and setters
if self.isProperty(elem):
return False
# ignore friend classes
if self.isFriendClass(elem):
return False
# ignore typedefs (can't test for them)
if self.isTypeDef(elem):
return False
if self.isVariable(elem) and self.visibility(elem) == "protected":
# protected variables can't be bound in SIP
return False
# check for members with special python doc notes (probably 'not available' or renamed methods, either way
# they should be safe to ignore as obviously some consideration has been given to Python bindings)
try:
detailed_sec = elem.find("detaileddescription")
for p in detailed_sec.iter("para"):
for s in p.iter("simplesect"):
for ps in s.iter("para"):
if ps.text and "python" in ps.text.lower():
return False
except:
pass
# ignore constructors and destructor, can't test for these
if self.isDestructor(elem) or self.isConstructor(elem):
return False
# ignore operators, also can't test
if self.isOperator(elem):
return False
# ignore deprecated members
if self.isDeprecated(elem):
return False
return True
def elemIsDocumentableMember(self, elem):
"""Tests whether an member should be included in Doxygen docs
:param elem: XML element for a class member
"""
# ignore variables (for now, eventually public/protected variables should be documented)
if self.isVariable(elem):
return False
# only public or protected members should be documented
if not self.visibility(elem) in ("public", "protected"):
return False
# ignore reimplemented methods
if self.isReimplementation(elem):
return False
# ignore friend classes
if self.isFriendClass(elem):
return False
# ignore destructor
if self.isDestructor(elem):
return False
if self.isConstructor(elem):
# ignore constructors with no arguments
try:
if re.match(
r"^\s*\(\s*\)\s*(?:=\s*(?:default|delete)\s*)?$",
elem.find("argsstring").text,
):
return False
except:
pass
# ignore copy, move constructors
name = elem.find("name").text
match = re.match(
r"^\s*\(\s*(?:const)?\s*"
+ name
+ r"\s*&{0,2}\s*(?:[a-zA-Z0-9_]+)?\s*\)\s*(?:=\s*(?:default|delete)\s*)?$",
elem.find("argsstring").text,
)
if match:
return False
name = elem.find("name")
# ignore certain obvious operators
if name.text in (
"operator=",
"operator==",
"operator!=",
"operator>=",
"operator>",
"operator<=",
"operator<",
"Q_ENUM",
):
return False
# ignore on_* slots
try:
if name.text.startswith("on_"):
return False
except:
pass
# ignore deprecated members
if self.isDeprecated(elem):
return False
return True
def visibility(self, elem):
"""Returns the visibility of a class or member
:param elem: XML element for a class or member
"""
try:
return elem.get("prot")
except:
return ""
def isVariable(self, member_elem):
"""Tests whether an member is a variable
:param member_elem: XML element for a class member
"""
try:
if member_elem.get("kind") == "variable":
return True
except:
pass
return False
def isProperty(self, member_elem):
"""Tests whether an member is a property
:param member_elem: XML element for a class member
"""
try:
if member_elem.get("kind") == "property":
return True
except:
pass
return False
def isDestructor(self, member_elem):
"""Tests whether an member is a destructor
:param member_elem: XML element for a class member
"""
try:
name = member_elem.find("name").text
if name.startswith("~"):
# destructor
return True
except:
pass
return False
def isConstructor(self, member_elem):
"""Tests whether an member is a constructor
:param member_elem: XML element for a class member
"""
try:
definition = member_elem.find("definition").text
name = member_elem.find("name").text
if f"{name}::{name}" in definition:
return True
if re.match(rf"{name}\s*\<\s*[a-zA-Z0-9_]+\s*\>\s*::{name}", definition):
return True
except (AttributeError, re.error):
pass
return False
def isOperator(self, member_elem):
"""Tests whether an member is an operator
:param member_elem: XML element for a class member
"""
try:
name = member_elem.find("name").text
if re.match(r"^operator\W.*", name):
return True
except:
pass
return False
def isFriendClass(self, member_elem):
"""Tests whether an member is a friend class
:param member_elem: XML element for a class member
"""
try:
if member_elem.get("kind") == "friend":
return True
except:
pass
return False
def isTypeDef(self, member_elem):
"""Tests whether an member is a type def
:param member_elem: XML element for a class member
"""
try:
if member_elem.get("kind") == "typedef":
return True
except:
pass
return False
def isReimplementation(self, member_elem):
"""Tests whether an member is a reimplementation
:param member_elem: XML element for a class member
"""
# use two different tests, as Doxygen will not detect reimplemented Qt methods
try:
if member_elem.find("reimplements") is not None:
return True
if " override" in member_elem.find("argsstring").text:
return True
if " final" in member_elem.find("argsstring").text.lower():
return True
except:
pass
return False
def isDeprecated(self, member_elem):
"""Tests whether an member is deprecated
:param member_elem: XML element for a class member
"""
# look for both Q_DECL_DEPRECATED and Doxygen deprecated tag
decl_deprecated = False
type_elem = member_elem.find("type")
try:
if "Q_DECL_DEPRECATED" in type_elem.text:
decl_deprecated = True
except:
pass
if b"Q_DECL_DEPRECATED" in ET.tostring(type_elem):
decl_deprecated = True
doxy_deprecated = False
has_description = True
try:
for p in member_elem.find("detaileddescription").iter("para"):
for s in p.iter("xrefsect"):
if (
s.find("xreftitle") is not None
and "Deprecated" in s.find("xreftitle").text
):
doxy_deprecated = True
if (
s.find("xrefdescription") is None
or s.find("xrefdescription").find("para") is None
):
has_description = False
break
except:
assert 0, member_elem.find("definition").text
if not decl_deprecated and not doxy_deprecated:
return False
if doxy_deprecated and not has_description:
assert (
has_description
), "Error: Missing description for deprecated method {}".format(
member_elem.find("definition").text
)
# only functions for now, but in future this should also apply for enums and variables
if member_elem.get("kind") in ("function", "variable"):
assert decl_deprecated, "Error: Missing Q_DECL_DEPRECATED for {}".format(
member_elem.find("definition").text
)
assert (
doxy_deprecated
), "Error: Missing Doxygen deprecated tag for {}".format(
member_elem.find("definition").text
)
return True
def memberIsDocumented(self, member_elem):
"""Tests whether an member has documentation
:param member_elem: XML element for a class member
"""
for doc_type in (
"inbodydescription",
"briefdescription",
"detaileddescription",
):
doc = member_elem.find(doc_type)
if doc is not None and list(doc):
return True
return False
def memberDocIsNonCompliant(self, member_elem):
"""Tests whether an member's documentation is non-compliant
:param member_elem: XML element for a class member
"""
def _check_compliance(elem):
for para in elem.iter("para"):
for sect in para.iter("simplesect"):
res = _check_compliance(sect)
if res:
return res
for t in para.itertext():
if doc_type == "briefdescription":
if t.strip().lower().startswith("getter"):
return 'Use "Returns the..." instead of "getter"'
if t.strip().lower().startswith("get "):
return 'Use "Gets..." (or better, "Returns ...") instead of "get ..."'
elif t.strip().lower().startswith("setter"):
return 'Use "Sets the..." instead of "setter"'
elif t.strip().lower().startswith("mutator"):
return 'Use "Sets the..." instead of "mutator for..."'
elif t.strip().lower().startswith("accessor"):
return 'Use "Returns the..." instead of "accessor for..."'
elif t.strip().lower().startswith("return "):
return 'Use "Returns the..." instead of "return ..."'
if re.search(r"\btodo\b", t.lower()) is not None:
return "Don't add TODO comments to public doxygen documentation. Leave these as c++ code comments only."
for doc_type in ["briefdescription", "detaileddescription"]:
doc = member_elem.find(doc_type)
if doc is not None:
res = _check_compliance(doc)
if res:
return res
return False
def checkForBrokenSeeAlsoLinks(self, elem):
"""
Checks for any broken 'see also' links
"""
broken = []
detailed_sec = elem.find("detaileddescription")
for p in detailed_sec.iter("para"):
for s in p.iter("simplesect"):
if s.get("kind") != "see":
continue
para = list(s.iter())[1]
if (
para.find("ref") is None
and para.text
and (not para.text.startswith("Q") or para.text.startswith("Qgs"))
):
broken.append(para.text)
return broken