docs/schedulers: add runopts to documentation (#374)

Summary:
This adds the runopts to the documentation for the schedulers. It uses a sphinx plugin to load the scheduler and generate the CLI help text which it places in a code block.

Pull Request resolved: #374

Test Plan:
```
make clean html
```
![Screenshot 2022-01-25 at 14-57-50 Slurm — PyTorch TorchX main documentation](https://user-images.githubusercontent.com/909104/151073867-f03ceffc-a4cf-44e4-96f6-2138f741b6de.png)
![Screenshot 2022-01-25 at 14-57-43 Kubernetes — PyTorch TorchX main documentation](https://user-images.githubusercontent.com/909104/151073869-f7b07523-4e20-4a95-b28b-b2c75f49f952.png)
![Screenshot 2022-01-25 at 14-57-31 Local — PyTorch TorchX main documentation](https://user-images.githubusercontent.com/909104/151073875-84f5f205-6cc7-49f8-8f8d-71e2b4d7ff4f.png)

Reviewed By: aivanou

Differential Revision: D33778344

Pulled By: d4l3k

fbshipit-source-id: cc8b52db083ecc199233ab2f419ebfd5d1e53c46

Author: d4l3k

docs/source/conf.py

@@ -61,6 +61,7 @@
     "sphinx.ext.autosectionlabel",
     "sphinx_gallery.gen_gallery",
     "compatibility",
+    "runopts",
     "nbsphinx",
     "IPython.sphinxext.ipython_console_highlighting",
 ]

docs/source/ext/compatibility.py

@@ -25,25 +25,6 @@ class CompatibilityDirective(SphinxDirective):
     has_content = True
 
     def run(self):
-        """
-        targetid = "todo-%d" % self.env.new_serialno("todo")
-        targetnode = nodes.target("", "", ids=[targetid])
-        todo_node = todo("\n".join(self.content))
-        todo_node += nodes.title(_("Todo"), _("Todo"))
-        self.state.nested_parse(self.content, self.content_offset, todo_node)
-        if not hasattr(self.env, "todo_all_todos"):
-            self.env.todo_all_todos = []
-
-        self.env.todo_all_todos.append(
-            {
-                "docname": self.env.docname,
-                "lineno": self.lineno,
-                "todo": todo_node.deepcopy(),
-                "target": targetnode,
-            }
-        )
-        """
-
         raw_content = "\n".join(self.content)
         args = yaml.safe_load(raw_content)
         fields = COMPATIBILITY_SETS[args["type"]]

docs/source/ext/runopts.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from pydoc import locate
+
+import yaml
+from docutils import nodes
+from sphinx.util.docutils import SphinxDirective
+
+
+class RunOptsDirective(SphinxDirective):
+    # this enables content in the directive
+    has_content = True
+
+    def run(self):
+        raw_content = "\n".join(self.content)
+        args = yaml.safe_load(raw_content)
+        cls = locate(args["class"])
+
+        body = nodes.literal_block(text=str(cls.run_opts(None)))
+        return [
+            body,
+        ]
+
+
+def setup(app):
+    app.add_directive("runopts", RunOptsDirective)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

torchx/schedulers/kubernetes_scheduler.py

@@ -331,6 +331,13 @@ class KubernetesScheduler(Scheduler):
         $ torchx status kubernetes://torchx_user/1234
         ...
 
+    **Config Options**
+
+    .. runopts::
+        class: torchx.schedulers.kubernetes_scheduler.KubernetesScheduler
+
+    **Compatibility**
+
     .. compatibility::
         type: scheduler
         features:

torchx/schedulers/local_scheduler.py

@@ -528,10 +528,16 @@ class LocalScheduler(Scheduler):
     .. note::
         The orphan cleanup only works if `LocalScheduler` is instantiated from the main thread.
 
+    **Config Options**
+
+    .. runopts::
+        class: torchx.schedulers.local_scheduler.LocalScheduler
+
+    **Compatibility**
+
     .. note::
-        Use this scheduler sparingly since an application that runs successfully
-        on a session backed by this scheduler may not work on an actual
-        production cluster using a different scheduler.
+        Due to scheduler differences jobs that run locally may not work when
+        using a different scheduler due to network or software dependencies.
 
     .. compatibility::
         type: scheduler

torchx/schedulers/slurm_scheduler.py

@@ -224,6 +224,13 @@ class SlurmScheduler(Scheduler):
         $ less slurm-1234.out
         ...
 
+    **Config Options**
+
+    .. runopts::
+        class: torchx.schedulers.slurm_scheduler.SlurmScheduler
+
+    **Compatibility**
+
     .. compatibility::
         type: scheduler
         features:
@@ -233,6 +240,7 @@ class SlurmScheduler(Scheduler):
             describe: |
                 Partial support. SlurmScheduler will return job and replica
                 status but does not provide the complete original AppSpec.
+
     """
 
     def __init__(self, session_name: str) -> None: