* [pbs-devel] [PATCH proxmox-backup] docs: add prototype sphinx extension for online help
@ 2020-07-21 11:25 Oguz Bektas
2020-07-21 11:44 ` Thomas Lamprecht
0 siblings, 1 reply; 3+ messages in thread
From: Oguz Bektas @ 2020-07-21 11:25 UTC (permalink / raw)
To: pbs-devel
goes through the sections in the documents and creates the
OnlineHelpInfo.js file from the explicitly defined section labels.
run "make scanrefs" in the docs folder to generate the output.
for example:
const pveOnlineHelpInfo = {
"create-backups": {
"link": "/docs/administration-guide.html#create-backups",
"title": "Creating Backups"
},
"pruning": {
"link": "/docs/administration-guide.html#pruning",
"title": "Pruning and Removing Backups"
},
"id1": {
"link": "/docs/administration-guide.html#id1",
"title": "Garbage Collection"
},
"pve-integration": {
"link": "/docs/administration-guide.html#pve-integration",
"title": "`Proxmox VE`_ integration"
},
"pxar-format": {
"link": "/docs/file-formats.html#pxar-format",
"title": "Proxmox File Archive Format (``.pxar``)"
},
"sysadmin-package-repositories": {
"link": "/docs/package-repositories.html#sysadmin-package-repositories",
"title": "Debian Package Repositories"
},
"fake-label": {
"link": "/docs/sysadmin.html#fake-label",
"title": "Hardware"
}
};
---
docs/Makefile | 4 ++
docs/_ext/proxmox-scanrefs.py | 90 +++++++++++++++++++++++++++++++++++
docs/conf.py | 7 ++-
3 files changed, 99 insertions(+), 2 deletions(-)
create mode 100644 docs/_ext/proxmox-scanrefs.py
diff --git a/docs/Makefile b/docs/Makefile
index 34ec8809..796861b1 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -68,6 +68,10 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst proxmox-backup-manage
proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst proxmox-backup-proxy/description.rst
rst2man $< >$@
+.PHONY: scanrefs
+scanrefs:
+ $(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
+
.PHONY: html
html: ${GENERATED_SYNOPSIS}
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
diff --git a/docs/_ext/proxmox-scanrefs.py b/docs/_ext/proxmox-scanrefs.py
new file mode 100644
index 00000000..bb0c532d
--- /dev/null
+++ b/docs/_ext/proxmox-scanrefs.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+# debugging stuff
+from pprint import pprint
+
+from typing import cast
+
+import json
+import re
+
+import os
+import io
+from docutils import nodes
+
+from sphinx.builders import Builder
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+# refs are added in the following manner before the title of a section (note underscore and newline before title):
+# .. _my-label:
+#
+# Section to ref
+# --------------
+#
+#
+# then referred to like (note missing underscore):
+# "see :ref:`my-label`"
+#
+# the benefit of using this is if a label is explicitly set for a section,
+# we can refer to it with this anchor even if the section name changes.
+#
+# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
+
+def setup(app):
+ logger.info('Mapping reference labels...')
+ app.add_builder(ReflabelMapper)
+ return {
+ 'version': '0.1',
+ 'parallel_read_safe': True,
+ 'parallel_write_safe': True,
+ }
+
+class ReflabelMapper(Builder):
+ name = 'proxmox-scanrefs'
+
+ def init(self):
+ self.docnames = []
+ self.env.online_help = {}
+
+ if not os.path.isdir(self.outdir):
+ os.mkdir(self.outdir)
+
+ self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
+ self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
+
+ def write_doc(self, docname, doctree):
+ for node in doctree.traverse(nodes.section):
+ #pprint(vars(node))
+
+ if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
+ filename = self.env.doc2path(docname)
+ filename_html = re.sub('.rst', '.html', filename)
+ labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
+ title = cast(nodes.title, node[0])
+ logger.info(f'traversing section {title.astext()}')
+ ref_name = getattr(title, 'rawsource', title.astext())
+
+ self.env.online_help[labelid]: dict = {'link': '', 'title': ''}
+ self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + f"#{labelid}"
+ self.env.online_help[labelid]['title'] = ref_name
+
+ return
+
+ def get_outdated_docs(self):
+ return 'all documents'
+
+ def prepare_writing(self, docnames):
+ return
+
+ def get_target_uri(self, docname, typ=None):
+ return ''
+
+ def finish(self):
+ # generate OnlineHelpInfo.js output
+ self.output.write("const pveOnlineHelpInfo = ")
+ self.output.write(json.dumps(self.env.online_help, indent=2))
+ self.output.write(";\n")
+ self.output.close()
+ return
diff --git a/docs/conf.py b/docs/conf.py
index 0c40ffb5..a9d45c2e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -18,9 +18,12 @@
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
-# import sys
+import sys
# sys.path.insert(0, os.path.abspath('.'))
+# custom extensions
+sys.path.append(os.path.abspath("./_ext"))
+
# -- Implement custom formatter for code-blocks ---------------------------
#
# * use smaller font
@@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
-extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
+extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
todo_link_only = True
--
2.20.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [pbs-devel] [PATCH proxmox-backup] docs: add prototype sphinx extension for online help
2020-07-21 11:25 [pbs-devel] [PATCH proxmox-backup] docs: add prototype sphinx extension for online help Oguz Bektas
@ 2020-07-21 11:44 ` Thomas Lamprecht
2020-07-21 12:31 ` Oguz Bektas
0 siblings, 1 reply; 3+ messages in thread
From: Thomas Lamprecht @ 2020-07-21 11:44 UTC (permalink / raw)
To: Proxmox Backup Server development discussion, Oguz Bektas
On 21.07.20 13:25, Oguz Bektas wrote:
> goes through the sections in the documents and creates the
> OnlineHelpInfo.js file from the explicitly defined section labels.
>
> run "make scanrefs" in the docs folder to generate the output.
note that this approach is not the same as PVE and is lacking label verification.
As said a few times, the current process looks like:
1. building docs: get all explicit defined labels, record them and their info in a
JSON structure.
2. building GUI: scan all javascript sources to get a list of all *used* onlineHelp
definitions. Use that list and the JSON doc info to get the target+title map
(const pveOnlineHelpInfo or const proxmoxOnlineHelpInfo). If the scanned JS list
includes something which is not included in the docs reference info -> error out
Your approach is to just generate the OnlineHelpInfo directly, no build check at all.
And fully working integration patch would be much nicer, i.e., I cannot use this yet
to get a onlineHelp button working.
We probably also want to have some sort of blockid normalization in
`Proxmox.Utils.get_help_info` so that we can use the same onlineHelp for both spelling
types, i.e., "chapter_zfs" vs. "chapter-zfs" or we enforce one variant.
>
> for example:
>
> const pveOnlineHelpInfo = {
> "create-backups": {
> "link": "/docs/administration-guide.html#create-backups",
> "title": "Creating Backups"
> },
> "pruning": {
> "link": "/docs/administration-guide.html#pruning",
> "title": "Pruning and Removing Backups"
> },
> "id1": {
> "link": "/docs/administration-guide.html#id1",
> "title": "Garbage Collection"
> },
> "pve-integration": {
> "link": "/docs/administration-guide.html#pve-integration",
> "title": "`Proxmox VE`_ integration"
> },
> "pxar-format": {
> "link": "/docs/file-formats.html#pxar-format",
> "title": "Proxmox File Archive Format (``.pxar``)"
> },
> "sysadmin-package-repositories": {
> "link": "/docs/package-repositories.html#sysadmin-package-repositories",
> "title": "Debian Package Repositories"
> },
> "fake-label": {
> "link": "/docs/sysadmin.html#fake-label",
> "title": "Hardware"
> }
> };
>
> ---
> docs/Makefile | 4 ++
> docs/_ext/proxmox-scanrefs.py | 90 +++++++++++++++++++++++++++++++++++
> docs/conf.py | 7 ++-
> 3 files changed, 99 insertions(+), 2 deletions(-)
> create mode 100644 docs/_ext/proxmox-scanrefs.py
>
> diff --git a/docs/Makefile b/docs/Makefile
> index 34ec8809..796861b1 100644
> --- a/docs/Makefile
> +++ b/docs/Makefile
> @@ -68,6 +68,10 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst proxmox-backup-manage
> proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst proxmox-backup-proxy/description.rst
> rst2man $< >$@
>
> +.PHONY: scanrefs
> +scanrefs:
> + $(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
> +
> .PHONY: html
> html: ${GENERATED_SYNOPSIS}
> $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
> diff --git a/docs/_ext/proxmox-scanrefs.py b/docs/_ext/proxmox-scanrefs.py
> new file mode 100644
> index 00000000..bb0c532d
> --- /dev/null
> +++ b/docs/_ext/proxmox-scanrefs.py
> @@ -0,0 +1,90 @@
> +#!/usr/bin/env python3
> +
> +# debugging stuff
> +from pprint import pprint
> +
> +from typing import cast
> +
> +import json
> +import re
> +
> +import os
> +import io
> +from docutils import nodes
> +
> +from sphinx.builders import Builder
> +from sphinx.util import logging
> +
> +logger = logging.getLogger(__name__)
> +
> +# refs are added in the following manner before the title of a section (note underscore and newline before title):
> +# .. _my-label:
> +#
> +# Section to ref
> +# --------------
> +#
> +#
> +# then referred to like (note missing underscore):
> +# "see :ref:`my-label`"
> +#
> +# the benefit of using this is if a label is explicitly set for a section,
> +# we can refer to it with this anchor even if the section name changes.
> +#
> +# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
> +
> +def setup(app):
> + logger.info('Mapping reference labels...')
> + app.add_builder(ReflabelMapper)
> + return {
> + 'version': '0.1',
> + 'parallel_read_safe': True,
> + 'parallel_write_safe': True,
> + }
> +
> +class ReflabelMapper(Builder):
> + name = 'proxmox-scanrefs'
> +
> + def init(self):
> + self.docnames = []
> + self.env.online_help = {}
> +
> + if not os.path.isdir(self.outdir):
> + os.mkdir(self.outdir)
> +
> + self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
> + self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
> +
> + def write_doc(self, docname, doctree):
> + for node in doctree.traverse(nodes.section):
> + #pprint(vars(node))
> +
> + if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
> + filename = self.env.doc2path(docname)
> + filename_html = re.sub('.rst', '.html', filename)
> + labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
> + title = cast(nodes.title, node[0])
> + logger.info(f'traversing section {title.astext()}')
> + ref_name = getattr(title, 'rawsource', title.astext())
> +
> + self.env.online_help[labelid]: dict = {'link': '', 'title': ''}
> + self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + f"#{labelid}"
> + self.env.online_help[labelid]['title'] = ref_name
> +
> + return
> +
> + def get_outdated_docs(self):
> + return 'all documents'
> +
> + def prepare_writing(self, docnames):
> + return
> +
> + def get_target_uri(self, docname, typ=None):
> + return ''
> +
> + def finish(self):
> + # generate OnlineHelpInfo.js output
> + self.output.write("const pveOnlineHelpInfo = ")
this is not pve? We support the general `proxmoxOnlineHelpInfo` name too, use that.
> + self.output.write(json.dumps(self.env.online_help, indent=2))
> + self.output.write(";\n")
> + self.output.close()
> + return
> diff --git a/docs/conf.py b/docs/conf.py
> index 0c40ffb5..a9d45c2e 100644
> --- a/docs/conf.py
> +++ b/docs/conf.py
> @@ -18,9 +18,12 @@
> # documentation root, use os.path.abspath to make it absolute, like shown here.
> #
> import os
> -# import sys
> +import sys
> # sys.path.insert(0, os.path.abspath('.'))
>
> +# custom extensions
> +sys.path.append(os.path.abspath("./_ext"))
> +
> # -- Implement custom formatter for code-blocks ---------------------------
> #
> # * use smaller font
> @@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
> # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
> # ones.
>
> -extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
> +extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
>
> todo_link_only = True
>
>
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [pbs-devel] [PATCH proxmox-backup] docs: add prototype sphinx extension for online help
2020-07-21 11:44 ` Thomas Lamprecht
@ 2020-07-21 12:31 ` Oguz Bektas
0 siblings, 0 replies; 3+ messages in thread
From: Oguz Bektas @ 2020-07-21 12:31 UTC (permalink / raw)
To: Thomas Lamprecht; +Cc: Proxmox Backup Server development discussion
On Tue, Jul 21, 2020 at 01:44:41PM +0200, Thomas Lamprecht wrote:
> On 21.07.20 13:25, Oguz Bektas wrote:
> > goes through the sections in the documents and creates the
> > OnlineHelpInfo.js file from the explicitly defined section labels.
> >
> > run "make scanrefs" in the docs folder to generate the output.
>
> note that this approach is not the same as PVE and is lacking label verification.
> As said a few times, the current process looks like:
>
> 1. building docs: get all explicit defined labels, record them and their info in a
> JSON structure.
> 2. building GUI: scan all javascript sources to get a list of all *used* onlineHelp
> definitions. Use that list and the JSON doc info to get the target+title map
> (const pveOnlineHelpInfo or const proxmoxOnlineHelpInfo). If the scanned JS list
> includes something which is not included in the docs reference info -> error out
>
> Your approach is to just generate the OnlineHelpInfo directly, no build check at all.
>
> And fully working integration patch would be much nicer, i.e., I cannot use this yet
> to get a onlineHelp button working.
>
> We probably also want to have some sort of blockid normalization in
> `Proxmox.Utils.get_help_info` so that we can use the same onlineHelp for both spelling
> types, i.e., "chapter_zfs" vs. "chapter-zfs" or we enforce one variant.
underscores automatically turn into dashes.
so if i explicitly define a label like this:
"""
.. _fake_label_with_underscore:
ZFS Administration
~~~~~~~~~~~~~~~~~~
blah blah
"""
and generate the file again:
# make scanrefs
output for that ref will be:
const proxmoxOnlineHelpInfo = {
...
"fake-label-with-underscore": {
"link": "/docs/sysadmin.html#fake-label-with-underscore",
"title": "ZFS Administration"
}
};
also if i say:
.. _fake-label-with-dash:
then it will be:
const proxmoxOnlineHelpInfo = {
...
"fake-label-with-dash": {
"link": "/docs/sysadmin.html#fake-label-with-dash",
"title": "ZFS Administration"
}
};
>
>
> >
> > for example:
> >
> > const pveOnlineHelpInfo = {
> > "create-backups": {
> > "link": "/docs/administration-guide.html#create-backups",
> > "title": "Creating Backups"
> > },
> > "pruning": {
> > "link": "/docs/administration-guide.html#pruning",
> > "title": "Pruning and Removing Backups"
> > },
> > "id1": {
> > "link": "/docs/administration-guide.html#id1",
> > "title": "Garbage Collection"
> > },
> > "pve-integration": {
> > "link": "/docs/administration-guide.html#pve-integration",
> > "title": "`Proxmox VE`_ integration"
> > },
> > "pxar-format": {
> > "link": "/docs/file-formats.html#pxar-format",
> > "title": "Proxmox File Archive Format (``.pxar``)"
> > },
> > "sysadmin-package-repositories": {
> > "link": "/docs/package-repositories.html#sysadmin-package-repositories",
> > "title": "Debian Package Repositories"
> > },
> > "fake-label": {
> > "link": "/docs/sysadmin.html#fake-label",
> > "title": "Hardware"
> > }
> > };
> >
> > ---
> > docs/Makefile | 4 ++
> > docs/_ext/proxmox-scanrefs.py | 90 +++++++++++++++++++++++++++++++++++
> > docs/conf.py | 7 ++-
> > 3 files changed, 99 insertions(+), 2 deletions(-)
> > create mode 100644 docs/_ext/proxmox-scanrefs.py
> >
> > diff --git a/docs/Makefile b/docs/Makefile
> > index 34ec8809..796861b1 100644
> > --- a/docs/Makefile
> > +++ b/docs/Makefile
> > @@ -68,6 +68,10 @@ proxmox-backup-manager.1: proxmox-backup-manager/man1.rst proxmox-backup-manage
> > proxmox-backup-proxy.1: proxmox-backup-proxy/man1.rst proxmox-backup-proxy/description.rst
> > rst2man $< >$@
> >
> > +.PHONY: scanrefs
> > +scanrefs:
> > + $(SPHINXBUILD) -b proxmox-scanrefs $(ALLSPHINXOPTS) $(BUILDDIR)/scanrefs
> > +
> > .PHONY: html
> > html: ${GENERATED_SYNOPSIS}
> > $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
> > diff --git a/docs/_ext/proxmox-scanrefs.py b/docs/_ext/proxmox-scanrefs.py
> > new file mode 100644
> > index 00000000..bb0c532d
> > --- /dev/null
> > +++ b/docs/_ext/proxmox-scanrefs.py
> > @@ -0,0 +1,90 @@
> > +#!/usr/bin/env python3
> > +
> > +# debugging stuff
> > +from pprint import pprint
> > +
> > +from typing import cast
> > +
> > +import json
> > +import re
> > +
> > +import os
> > +import io
> > +from docutils import nodes
> > +
> > +from sphinx.builders import Builder
> > +from sphinx.util import logging
> > +
> > +logger = logging.getLogger(__name__)
> > +
> > +# refs are added in the following manner before the title of a section (note underscore and newline before title):
> > +# .. _my-label:
> > +#
> > +# Section to ref
> > +# --------------
> > +#
> > +#
> > +# then referred to like (note missing underscore):
> > +# "see :ref:`my-label`"
> > +#
> > +# the benefit of using this is if a label is explicitly set for a section,
> > +# we can refer to it with this anchor even if the section name changes.
> > +#
> > +# see https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-ref
> > +
> > +def setup(app):
> > + logger.info('Mapping reference labels...')
> > + app.add_builder(ReflabelMapper)
> > + return {
> > + 'version': '0.1',
> > + 'parallel_read_safe': True,
> > + 'parallel_write_safe': True,
> > + }
> > +
> > +class ReflabelMapper(Builder):
> > + name = 'proxmox-scanrefs'
> > +
> > + def init(self):
> > + self.docnames = []
> > + self.env.online_help = {}
> > +
> > + if not os.path.isdir(self.outdir):
> > + os.mkdir(self.outdir)
> > +
> > + self.output_filename = os.path.join(self.outdir, 'OnlineHelpInfo.js')
> > + self.output = io.open(self.output_filename, 'w', encoding='UTF-8')
> > +
> > + def write_doc(self, docname, doctree):
> > + for node in doctree.traverse(nodes.section):
> > + #pprint(vars(node))
> > +
> > + if hasattr(node, 'expect_referenced_by_id') and len(node['ids']) > 1: # explicit labels
> > + filename = self.env.doc2path(docname)
> > + filename_html = re.sub('.rst', '.html', filename)
> > + labelid = node['ids'][1] # [0] is predefined by sphinx, we need [1] for explicit ones
> > + title = cast(nodes.title, node[0])
> > + logger.info(f'traversing section {title.astext()}')
> > + ref_name = getattr(title, 'rawsource', title.astext())
> > +
> > + self.env.online_help[labelid]: dict = {'link': '', 'title': ''}
> > + self.env.online_help[labelid]['link'] = "/docs/" + os.path.basename(filename_html) + f"#{labelid}"
> > + self.env.online_help[labelid]['title'] = ref_name
> > +
> > + return
> > +
> > + def get_outdated_docs(self):
> > + return 'all documents'
> > +
> > + def prepare_writing(self, docnames):
> > + return
> > +
> > + def get_target_uri(self, docname, typ=None):
> > + return ''
> > +
> > + def finish(self):
> > + # generate OnlineHelpInfo.js output
> > + self.output.write("const pveOnlineHelpInfo = ")
>
> this is not pve? We support the general `proxmoxOnlineHelpInfo` name too, use that.
>
> > + self.output.write(json.dumps(self.env.online_help, indent=2))
> > + self.output.write(";\n")
> > + self.output.close()
> > + return
> > diff --git a/docs/conf.py b/docs/conf.py
> > index 0c40ffb5..a9d45c2e 100644
> > --- a/docs/conf.py
> > +++ b/docs/conf.py
> > @@ -18,9 +18,12 @@
> > # documentation root, use os.path.abspath to make it absolute, like shown here.
> > #
> > import os
> > -# import sys
> > +import sys
> > # sys.path.insert(0, os.path.abspath('.'))
> >
> > +# custom extensions
> > +sys.path.append(os.path.abspath("./_ext"))
> > +
> > # -- Implement custom formatter for code-blocks ---------------------------
> > #
> > # * use smaller font
> > @@ -46,7 +49,7 @@ PygmentsBridge.latex_formatter = CustomLatexFormatter
> > # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
> > # ones.
> >
> > -extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo"]
> > +extensions = ["sphinx.ext.graphviz", "sphinx.ext.todo", "proxmox-scanrefs"]
> >
> > todo_link_only = True
> >
> >
>
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2020-07-21 12:31 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-07-21 11:25 [pbs-devel] [PATCH proxmox-backup] docs: add prototype sphinx extension for online help Oguz Bektas
2020-07-21 11:44 ` Thomas Lamprecht
2020-07-21 12:31 ` Oguz Bektas
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox