[pbs-devel] [PATCH v4 proxmox-backup] docs: add prototype sphinx extension for online help

[pbs-devel] [PATCH v4 proxmox-backup] docs: add prototype sphinx extension for online help

[Oguz Bektas]

goes through the sections in the documents and creates the
OnlineHelpInfo.js file from the explicitly defined section labels which
are used in the js files with 'onlineHelp' variable.
---
v3->v4:
* s/pbs-docs/docs for the documentation index


 docs/Makefile                 |   7 +-
 docs/_ext/proxmox-scanrefs.py | 133 ++++++++++++++++++++++++++++++++++
 docs/conf.py                  |   7 +-
 docs/local-zfs.rst            |   3 +
 www/Makefile                  |   4 +
 www/OnlineHelpInfo.js         |  14 ++--
 6 files changed, 160 insertions(+), 8 deletions(-)
 create mode 100644 docs/_ext/proxmox-scanrefs.py

diff --git a/docs/Makefile b/docs/Makefile
index 34ec8809..8d1e8a1b 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -28,7 +28,6 @@ COMPILEDIR := ../target/debug
 SPHINXOPTS    += -t devbuild
 endif
 
-
 # Sphinx internal variables.
 ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) .
 
@@ -68,6 +67,12 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst  proxmox-backup-manage
 proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst  proxmox-backup-proxy/description.rst
 	rst2man $< >$@
 
+.PHONY: onlinehelpinfo
+onlinehelpinfo:
+	@echo "Generating OnlineHelpInfo.js..."
+	$(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
+	@echo "Build finished. OnlineHelpInfo.js is in $(BUILDDIR)/scanrefs."
+
 .PHONY: html
 html: ${GENERATED_SYNOPSIS}
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
diff --git a/docs/_ext/proxmox-scanrefs.py b/docs/_ext/proxmox-scanrefs.py
new file mode 100644
index 00000000..dd88f462
--- /dev/null
+++ b/docs/_ext/proxmox-scanrefs.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+# debugging stuff
+from pprint import pprint
+
+from typing import cast
+
+import json
+import re
+
+import os
+import io
+from docutils import nodes
+
+from sphinx.builders import Builder
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+# refs are added in the following manner before the title of a section (note underscore and newline before title):
+# .. _my-label:
+#
+# Section to ref
+# --------------
+#
+#
+# then referred to like (note missing underscore):
+# "see :ref:`my-label`"
+#
+# the benefit of using this is if a label is explicitly set for a section,
+# we can refer to it with this anchor #my-label in the html,
+# even if the section name changes.
+#
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+
+def scan_extjs_files(wwwdir="../www"): # a bit rough i know, but we can optimize later
+    js_files = []
+    used_anchors = []
+    logger.info("scanning extjs files for onlineHelp definitions")
+    for root, dirs, files in os.walk("{}".format(wwwdir)):
+        #print(root, dirs, files)
+        for filename in files:
+            if filename.endswith('.js'):
+                js_files.append(os.path.join(root, filename))
+    for js_file in js_files:
+        fd = open(js_file).read()
+        match = re.search("onlineHelp:\s*[\'\"](.*?)[\'\"]", fd) # match object is tuple
+        if match:
+            anchor = match.groups()[0]
+            anchor = re.sub('_', '-', anchor) # normalize labels
+            logger.info("found onlineHelp: {} in {}".format(anchor, js_file))
+            used_anchors.append(anchor)
+    return used_anchors
+
+
+def setup(app):
+    logger.info('Mapping reference labels...')
+    app.add_builder(ReflabelMapper)
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }
+
+class ReflabelMapper(Builder):
+    name = 'proxmox-scanrefs'
+
+    def init(self):
+        self.docnames = []
+        self.env.online_help = {}
+        self.env.online_help['pbs_documentation_index'] = {
+            'link': '/docs/index.html',
+            'title': 'Proxmox Backup Server Documentation Index',
+        }
+        self.env.used_anchors = scan_extjs_files()
+
+        if not os.path.isdir(self.outdir):
+            os.mkdir(self.outdir)
+
+        self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
+        self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
+
+    def write_doc(self, docname, doctree):
+            for node in doctree.traverse(nodes.section):
+                #pprint(vars(node))
+
+                if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
+                    filename = self.env.doc2path(docname)
+                    filename_html = re.sub('.rst', '.html', filename)
+                    labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
+                    title = cast(nodes.title, node[0])
+                    logger.info('traversing section {}'.format(title.astext()))
+                    ref_name = getattr(title, 'rawsource', title.astext())
+
+                    self.env.online_help[labelid] = {'link': '', 'title': ''}
+                    self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + "#{}".format(labelid)
+                    self.env.online_help[labelid]['title'] = ref_name
+
+            return
+
+
+    def get_outdated_docs(self):
+        return 'all documents'
+
+    def prepare_writing(self, docnames):
+        return
+
+    def get_target_uri(self, docname, typ=None):
+        return ''
+
+    def validate_anchors(self):
+        #pprint(self.env.online_help)
+        to_remove = []
+        for anchor in self.env.used_anchors:
+            if anchor not in self.env.online_help:
+                logger.info("[-] anchor {} is missing from onlinehelp!".format(anchor))
+        for anchor in self.env.online_help:
+            if anchor not in self.env.used_anchors and anchor != 'pbs_documentation_index':
+                logger.info("[*] anchor {} not used! deleting...".format(anchor))
+                to_remove.append(anchor)
+        for anchor in to_remove:
+            self.env.online_help.pop(anchor, None)
+        return
+
+    def finish(self):
+        # generate OnlineHelpInfo.js output
+        self.validate_anchors()
+
+        self.output.write("const proxmoxOnlineHelpInfo = ")
+        self.output.write(json.dumps(self.env.online_help, indent=2))
+        self.output.write(";\n")
+        self.output.close()
+        return
diff --git a/docs/conf.py b/docs/conf.py
index 0c40ffb5..a9d45c2e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -18,9 +18,12 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-# import sys
+import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
+# custom extensions
+sys.path.append(os.path.abspath("./_ext"))
+
 # -- Implement custom formatter for code-blocks ---------------------------
 #
 # * use smaller font
@@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 
-extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
+extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
 
 todo_link_only = True
 
diff --git a/docs/local-zfs.rst b/docs/local-zfs.rst
index 41633fd8..cd9d5315 100644
--- a/docs/local-zfs.rst
+++ b/docs/local-zfs.rst
@@ -1,3 +1,6 @@
+
+.. _chapter-zfs:
+
 ZFS on Linux
 ------------
 
diff --git a/www/Makefile b/www/Makefile
index edce8cb3..992c4957 100644
--- a/www/Makefile
+++ b/www/Makefile
@@ -54,6 +54,10 @@ all: js/proxmox-backup-gui.js css/ext6-pbs.css
 js:
 	mkdir js
 
+OnlineHelpInfo.js:
+	$(MAKE) -C ../docs onlinehelpinfo
+	mv ../docs/output/scanrefs/OnlineHelpInfo.js .
+
 js/proxmox-backup-gui.js: js OnlineHelpInfo.js ${JSSRC}
 	cat OnlineHelpInfo.js ${JSSRC} >$@.tmp
 	mv $@.tmp $@
diff --git a/www/OnlineHelpInfo.js b/www/OnlineHelpInfo.js
index bb48bab6..c7c6363f 100644
--- a/www/OnlineHelpInfo.js
+++ b/www/OnlineHelpInfo.js
@@ -1,6 +1,10 @@
-var proxmoxOnlineHelpInfo = {
-    "pbs_documentation_index" : {
-	"link" : "/pbs-docs/index.html",
-	"title" : "Proxmox Backup Server Documentation Index"
-    }
+const proxmoxOnlineHelpInfo = {
+  "pbs_documentation_index": {
+    "link": "/pbs-docs/index.html",
+    "title": "Proxmox Backup Server Documentation Index"
+  },
+  "chapter-zfs": {
+    "link": "/docs/sysadmin.html#chapter-zfs",
+    "title": "ZFS on Linux"
+  }
 };
-- 
2.20.1

[pbs-devel] applied: [PATCH v4 proxmox-backup] docs: add prototype sphinx extension for online help

[Thomas Lamprecht]

On 02.09.20 14:22, Oguz Bektas wrote:
> goes through the sections in the documents and creates the
> OnlineHelpInfo.js file from the explicitly defined section labels which
> are used in the js files with 'onlineHelp' variable.
> ---
> v3->v4:
> * s/pbs-docs/docs for the documentation index
> 
> 
>  docs/Makefile                 |   7 +-
>  docs/_ext/proxmox-scanrefs.py | 133 ++++++++++++++++++++++++++++++++++
>  docs/conf.py                  |   7 +-
>  docs/local-zfs.rst            |   3 +
>  www/Makefile                  |   4 +
>  www/OnlineHelpInfo.js         |  14 ++--
>  6 files changed, 160 insertions(+), 8 deletions(-)
>  create mode 100644 docs/_ext/proxmox-scanrefs.py
> 
>

applied, thanks! I made the OnlineHelpInfo.js a .PHONY target and re-generated
it, FYI.