[BD-21] Sphinx extension and configuration file for setting annotations

Similar to the `featuretoggles` extension, this adds a `settings` Sphinx
extension, along with a draft setting annotation configuration file.

Author: regisb

CHANGELOG.rst

@@ -11,6 +11,11 @@ Change Log
 
 .. There should always be an "Unreleased" section for changes pending release.
 
+[0.8.0] - 2020-09-10
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Add ``settings`` Sphinx extension with setting annotation configuration file
+
 [0.7.0] - 2020-09-07
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

code_annotations/__init__.py

@@ -2,4 +2,4 @@
 Extensible tools for parsing annotations in codebases.
 """
 
-__version__ = '0.7.0'
+__version__ = '0.8.0'

code_annotations/config_and_tools/config/setting_annotations.yaml

@@ -0,0 +1,12 @@
+source_path: ./
+report_path: reports
+safelist_path: .annotation_safe_list.yml
+annotations:
+  setting:
+    - ".. setting_name:":
+    - ".. setting_default:":
+    - ".. setting_description:":
+    - ".. setting_warning:":
+extensions:
+    python:
+        - py

code_annotations/config_and_tools/sphinx/extensions/__init__.py

@@ -0,0 +1,3 @@
+"""
+Sphinx extensions for generating human-readable documentation from code annotations.
+"""

code_annotations/config_and_tools/sphinx/extensions/base.py

@@ -0,0 +1,54 @@
+"""
+Base utilities for building annotation-based Sphinx extensions.
+"""
+
+
+from code_annotations.base import AnnotationConfig
+from code_annotations.find_static import StaticSearch
+
+
+def find_annotations(source_path, config_path, group_by_key):
+    """
+    Find the feature toggles as defined in the configuration file.
+
+    Return:
+        toggles (dict): feature toggles indexed by name.
+    """
+    config = AnnotationConfig(
+        config_path, verbosity=0, source_path_override=source_path
+    )
+    results = StaticSearch(config).search()
+
+    toggles = {}
+    current_entry = {}
+    for filename, entries in results.items():
+        for entry in entries:
+            key = entry["annotation_token"]
+            value = entry["annotation_data"]
+            if key == group_by_key:
+                toggle_name = value
+                toggles[toggle_name] = {
+                    "filename": filename,
+                    "line_number": entry["line_number"],
+                }
+                current_entry = toggles[toggle_name]
+            else:
+                current_entry[key] = value
+
+    return toggles
+
+
+def quote_value(value):
+    """
+    Quote a Python object if it is string-like.
+    """
+    if value in ("True", "False", "None"):
+        return str(value)
+    try:
+        float(value)
+        return str(value)
+    except ValueError:
+        pass
+    if isinstance(value, str):
+        return '"{}"'.format(value)
+    return str(value)

code_annotations/config_and_tools/sphinx/extensions/featuretoggles.py

@@ -5,11 +5,11 @@
 
 import pkg_resources
 
-from code_annotations.base import AnnotationConfig
-from code_annotations.find_static import StaticSearch
 from docutils import nodes
 from sphinx.util.docutils import SphinxDirective
 
+from .base import find_annotations, quote_value
+
 
 def find_feature_toggles(source_path):
     """
@@ -22,28 +22,7 @@ def find_feature_toggles(source_path):
         "code_annotations",
         os.path.join("config_and_tools", "config", "feature_toggle_annotations.yaml"),
     )
-    config = AnnotationConfig(
-        config_path, verbosity=0, source_path_override=source_path
-    )
-    results = StaticSearch(config).search()
-
-    toggles = {}
-    current_entry = {}
-    for filename, entries in results.items():
-        for entry in entries:
-            key = entry["annotation_token"]
-            value = entry["annotation_data"]
-            if key == ".. toggle_name:":
-                toggle_name = value
-                toggles[toggle_name] = {
-                    "filename": filename,
-                    "line_number": entry["line_number"],
-                }
-                current_entry = toggles[toggle_name]
-            else:
-                current_entry[key] = value
-
-    return toggles
+    return find_annotations(source_path, config_path, ".. toggle_name:")
 
 
 class FeatureToggles(SphinxDirective):
@@ -121,22 +100,6 @@ def iter_nodes(self):
                 )
 
 
-def quote_value(value):
-    """
-    Quote a Python object if it is string-like.
-    """
-    if value in ("True", "False", "None"):
-        return str(value)
-    try:
-        float(value)
-        return str(value)
-    except ValueError:
-        pass
-    if isinstance(value, str):
-        return '"{}"'.format(value)
-    return str(value)
-
-
 def setup(app):
     """
     Declare the Sphinx extension.

code_annotations/config_and_tools/sphinx/extensions/settings.py

@@ -0,0 +1,125 @@
+"""
+Sphinx extension for viewing (non-toggle) setting annotations.
+"""
+import os
+
+import pkg_resources
+
+from docutils import nodes
+from docutils.parsers.rst import directives
+from sphinx.util.docutils import SphinxDirective
+
+from .base import find_annotations, quote_value
+
+
+def find_settings(source_path):
+    """
+    Find the Django settings as defined in the configuration file.
+
+    Return:
+        settings (dict): Django settings indexed by name.
+    """
+    config_path = pkg_resources.resource_filename(
+        "code_annotations",
+        os.path.join("config_and_tools", "config", "setting_annotations.yaml"),
+    )
+    return find_annotations(source_path, config_path, ".. setting_name:")
+
+
+class Settings(SphinxDirective):
+    """
+    Sphinx directive to document Django settings in a single documentation page.
+
+    Use this directive as follows::
+
+        .. settings::
+            :folder_path: lms/envs/common.py
+
+    This directive supports the following configuration parameters:
+
+    - ``settings_source_path``: absolute path to the repository file tree. E.g:
+
+        settings_source_path = os.path.join(os.path.dirname(__file__), "..", "..")
+
+    - ``settings_repo_url``: Github repository where the code is hosted. E.g:
+
+        settings_repo_url = "https://github.com/edx/myrepo"
+
+    - ``settings_repo_version``: current version of the git repository. E.g:
+
+        import git
+        try:
+            repo = git.Repo(search_parent_directories=True)
+            settings_repo_version = repo.head.object.hexsha
+        except git.InvalidGitRepositoryError:
+            settings_repo_version = "master"
+    """
+
+    required_arguments = 0
+    optional_arguments = 1
+    option_spec = {"folder_path": directives.unchanged}
+
+    def run(self):
+        """
+        Public interface of the Directive class.
+
+        Return:
+            nodes (list): nodes to be appended to the resulting document.
+        """
+        toggle_nodes = list(self.iter_nodes())
+        return [nodes.section("", *toggle_nodes, ids=["settings"])]
+
+    def iter_nodes(self):
+        """
+        Iterate on the docutils nodes generated by this directive.
+        """
+        source_path = os.path.join(
+            self.env.config.settings_source_path, self.options.get("folder_path", "")
+        )
+        settings = find_settings(source_path)
+        for setting_name in sorted(settings):
+            setting = settings[setting_name]
+            yield nodes.title(text=setting_name)
+            setting_default_value = setting.get(".. setting_default:", "Not defined")
+            setting_default_node = nodes.literal(
+                text=quote_value(setting_default_value)
+            )
+            yield nodes.paragraph("", "Default: ", setting_default_node)
+            yield nodes.paragraph(
+                "",
+                "Source: ",
+                nodes.reference(
+                    text="{} (line {})".format(
+                        setting["filename"], setting["line_number"]
+                    ),
+                    refuri="{}/blob/{}/{}#L{}".format(
+                        self.env.config.settings_repo_url,
+                        self.env.config.settings_repo_version,
+                        setting["filename"],
+                        setting["line_number"],
+                    ),
+                ),
+            )
+            yield nodes.paragraph(text=setting.get(".. setting_description:", ""))
+            if setting.get(".. setting_warning:") not in (None, "None", "n/a", "N/A"):
+                yield nodes.warning(
+                    "", nodes.paragraph("", setting[".. setting_warning:"])
+                )
+
+
+def setup(app):
+    """
+    Declare the Sphinx extension.
+    """
+    app.add_config_value(
+        "settings_source_path", os.path.abspath(".."), "env",
+    )
+    app.add_config_value("settings_repo_url", "", "env")
+    app.add_config_value("settings_repo_version", "master", "env")
+    app.add_directive("settings", Settings)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

docs/sphinx_extensions.rst

@@ -1,10 +1,14 @@
 Sphinx extensions
 -----------------
 
+This package can be used to document a couple things in your code case. The following Sphinx extensions will help you generate human-readable docs by parsing the source code, instead of importing it. This allows you to document your code base in any language, without having to install all its dependencies.
+
+.. _sphinx_featuretoggles:
+
 ``featuretoggles``
 ==================
 
-This package can be used to document the feature toggles in your code base. To do so,
+Feature toggles are an Open edX mechanism by which features can be individually enabled or disabled. To document these feature toggles,
 add the following to your ``conf.py``::
 
     extensions = ["code_annotations.config_and_tools.sphinx.extensions.featuretoggles"]
@@ -20,3 +24,22 @@ add the following to your ``conf.py``::
 Then, in an ``.rst`` file::
 
     .. featuretoggles::
+
+.. _sphinx_settings:
+
+``settings``
+============
+
+This package also comes with tooling to annotate Django settings. Settings that should be annotated include non-standard Django settings, and settings that do not correspond to feature toggles (in which case feature toggle annotations should be used instead). Similar to feature toggles, Django setting annotations can also be parsed to generate human-readable documentation. Add the following to your ``conf.py``::
+
+    extensions = ["code_annotations.config_and_tools.sphinx.extensions.settings"]
+
+Define the following variables, just like for the :ref:`featuretoggles <sphinx_featuretoggles>` extension::
+
+    settings_source_path = ...
+    settings_repo_url = ...
+
+Then, in an ``.rst`` file::
+
+    .. settings::
+        :folder_path: path/to/settings.py