Generic config module (pytorch#508)

Summary:
Pull Request resolved: pytorch#508

Extract generic config module and decouple storage concerns from configuration data.

- This should allow adding new configuration options besides INI
- All the client code should work before, but we are moving away from runner specific logic.

Github issue (3rd task) pytorch#368

Differential Revision:
D36855026

LaMa Project: L1123150

fbshipit-source-id: 3562bf258270c661354ea58264cb099037b1d984

Author: kurman

torchx/cli/argparse_util.py

@@ -8,8 +8,7 @@
 from pathlib import Path
 from typing import Any, Dict, Optional, Sequence, Text
 
-from torchx.runner import config
-
+from torchx.config.config import get_config_store
 
 CONFIG_DIRS = [str(Path.home()), str(Path.cwd())]
 
@@ -35,13 +34,11 @@ def __init__(
         default: Any = None,
         **kwargs: Any,
     ) -> None:
+        conf_data = get_config_store(CONFIG_DIRS).get(collection=subcmd, label="cli")
+        conf_data_dict = dict(conf_data.items())
         cfg = self._subcmd_configs.setdefault(
             subcmd,
-            config.get_configs(
-                prefix="cli",
-                name=subcmd,
-                dirs=CONFIG_DIRS,
-            ),
+            conf_data_dict,
         )
 
         # if found in .torchxconfig make it the default for this argument

torchx/cli/cmd_configure.py

@@ -10,7 +10,7 @@
 import sys
 
 from torchx.cli.cmd_base import SubCommand
-from torchx.runner.config import dump
+from torchx.config import seed_config_store_with_schedulers
 from torchx.schedulers import get_schedulers
 
 
@@ -47,7 +47,11 @@ def run(self, args: argparse.Namespace) -> None:
         required_only = not args.all
 
         if args.print:
-            dump(f=sys.stdout, schedulers=schedulers, required_only=required_only)
+            seed_config_store_with_schedulers(
+                f=sys.stdout, schedulers=schedulers, required_only=required_only
+            )
         else:
             with open(".torchxconfig", "w") as f:
-                dump(f=f, schedulers=schedulers, required_only=required_only)
+                seed_config_store_with_schedulers(
+                    f=f, schedulers=schedulers, required_only=required_only
+                )

torchx/cli/cmd_run.py

@@ -19,8 +19,8 @@
 from torchx.cli.argparse_util import CONFIG_DIRS, torchxconfig_run
 from torchx.cli.cmd_base import SubCommand
 from torchx.cli.cmd_log import get_logs
-from torchx.runner import config, get_runner, Runner
-from torchx.runner.config import load_sections
+from torchx.config import get_config_store
+from torchx.runner import get_runner, Runner
 from torchx.schedulers import get_default_scheduler_name, get_scheduler_factories
 from torchx.specs.finder import (
     _Component,
@@ -63,7 +63,11 @@ def _parse_component_name_and_args(
                 looks like an option (e.g. starts with "-")
 
     """
-    component = config.get_config(prefix="cli", name="run", key="component", dirs=dirs)
+    component = get_config_store(dirs).get_key(
+        collection="run", key="component", label="cli"
+    )
+
+    component = str(component) if component else None
     component_args = []
 
     # make a copy of the input list to guard against side-effects
@@ -177,7 +181,12 @@ def _run(self, runner: Runner, args: argparse.Namespace) -> None:
         run_opts = runner.run_opts()
         scheduler_opts = run_opts[args.scheduler]
         cfg = scheduler_opts.cfg_from_str(args.scheduler_args)
-        config.apply(scheduler=args.scheduler, cfg=cfg, dirs=CONFIG_DIRS)
+
+        type_hints = {x[0]: x[1].opt_type for x in scheduler_opts}
+        config_data = get_config_store(CONFIG_DIRS).get(
+            args.scheduler, type_hints=type_hints
+        )
+        cfg = config_data.apply_to(cfg)
 
         component, component_args = _parse_component_name_and_args(
             args.component_name_and_args,
@@ -226,18 +235,22 @@ def _run(self, runner: Runner, args: argparse.Namespace) -> None:
             logger.error(error_msg)
             sys.exit(1)
         except specs.InvalidRunConfigException as e:
+            logger.error(e)
             error_msg = (
                 f"Scheduler arg is incorrect or missing required option: `{e.cfg_key}`\n"
                 f"Run `torchx runopts` to check configuration for `{args.scheduler}` scheduler\n"
                 f"Use `-cfg` to specify run cfg as `key1=value1,key2=value2` pair\n"
-                "of setup `.torchxconfig` file, see: https://pytorch.org/torchx/main/experimental/runner.config.html"
+                "of setup `.torchxconfig` file, see: https://pytorch.org/torchx/main/runner.config.html"
             )
             logger.error(error_msg)
             sys.exit(1)
 
     def run(self, args: argparse.Namespace) -> None:
         os.environ["TORCHX_CONTEXT_NAME"] = os.getenv("TORCHX_CONTEXT_NAME", "cli_run")
-        component_defaults = load_sections(prefix="component", dirs=CONFIG_DIRS)
+        config_data = get_config_store(CONFIG_DIRS).get_all(label="component")
+        component_defaults = {}
+        for conf in config_data:
+            component_defaults[conf.coll_name] = dict(conf)
         with get_runner(component_defaults=component_defaults) as runner:
             self._run(runner, args)
 

torchx/config/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .config import (  # noqa
+    ConfigData,
+    ConfigStore,
+    get_config_store,
+    IniConfigStore,
+    seed_config_store_with_schedulers,
+)