Algo Benchmark Library card catalog

Change-Id: I4ca692cd92ddbee95d0d2aeed5a4027fcd0f2aa4

Author: mpharrigan

recirq/algo_benchmark_library.py

@@ -0,0 +1,83 @@
+import dataclasses
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Type, Callable, List, Tuple
+
+import pandas as pd
+
+from cirq_google.workflow import ExecutableSpec, QuantumExecutableGroup
+
+
+@dataclass
+class AlgoBenchmark:
+    domain: str
+    """The problem domain. Usually a high-level ReCirq module."""
+
+    name: str
+    """The benchmark name. Must be unique within the domain."""
+
+    executable_family: str
+    """A globally unique identifier for this AlgoBenchmark.
+    
+    This should match up with this Benchmark's `spec_class.executable_family`.
+    
+    By convention, the executable family is the fully-qualified leaf-module where the code
+    for this AlgoBenchmark lives.
+    """
+
+    spec_class: Type[ExecutableSpec]
+    """The ExecutableSpec subclass for this AlgoBenchmark."""
+
+    data_class: Type
+    """The class which can contain ETL-ed data for this AlgoBenchmark."""
+
+    gen_func: Callable[[...], QuantumExecutableGroup]
+    """The function that returns a QuantumExecutableGroup for a given Config."""
+
+    configs: List['BenchmarkConfig']
+
+    def as_strings(self):
+        ret = {k: str(v) for k, v in dataclasses.asdict(self).items()}
+        ret['spec_class'] = self.spec_class.__name__
+        ret['data_class'] = self.data_class.__name__
+        ret['gen_func'] = self.gen_func.__name__
+        return ret
+
+
+@dataclass
+class BenchmarkConfig:
+    short_name: str
+    """The short name for this config. Unique within an AlgoBenchmark."""
+
+    full_name: str
+    """A globally unique name for this config."""
+
+    gen_script: str
+    """The script filename that generates the QuantumExecutableGroup for this Config."""
+
+    run_scripts: List[str]
+    """A list of script filenames that execute (or can execute) this config."""
+
+
+BENCHMARKS = [
+]
+
+
+@lru_cache()
+def get_all_algo_configs() -> List[Tuple[AlgoBenchmark, BenchmarkConfig]]:
+    ret = []
+    for algo in BENCHMARKS:
+        for config in algo.configs:
+            ret.append((algo, config))
+    return ret
+
+
+def main():
+    df = pd.DataFrame([algo.as_strings() for algo in BENCHMARKS]).set_index('executable_family')
+    pd.set_option('display.width', None)
+    pd.set_option('display.max_colwidth', None)
+    print(df.transpose())
+
+
+if __name__ == '__main__':
+    main()

recirq/algo_benchmark_library_test.py

@@ -0,0 +1,70 @@
+import os
+from importlib import import_module
+from types import ModuleType
+
+import pytest
+
+from recirq.algo_benchmark_library import BENCHMARKS, get_all_algo_configs
+
+RECIRQ_DIR = os.path.abspath(os.path.dirname(__file__) + '/../')
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_domain(algo):
+    # By convention, the domain should be a recirq module.
+    assert algo.domain.startswith('recirq.')
+    mod = import_module(algo.domain)
+    assert isinstance(mod, ModuleType)
+
+
+def test_benchmark_name_unique_in_domain():
+    # In a given domain, all benchmark names should be unique
+    pairs = [(algo.domain, algo.name) for algo in BENCHMARKS]
+    assert len(set(pairs)) == len(pairs)
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_executable_family_is_formulaic(algo):
+    # Check consistency in the AlgoBenchmark dataclass:
+    assert algo.executable_family == algo.spec_class.executable_family
+
+    # By convention, we set this to be the module name. By further convention,
+    # {algo.domain}.{algo.name} should be the module name.
+    assert algo.executable_family == f'{algo.domain}.{algo.name}'
+
+    # Check the convention that it should give a module
+    mod = import_module(algo.executable_family)
+    assert isinstance(mod, ModuleType)
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_classes_and_funcs(algo):
+    # The various class objects should exist in the module
+    mod = import_module(algo.executable_family)
+    assert algo.spec_class == getattr(mod, algo.spec_class.__name__)
+    assert algo.data_class == getattr(mod, algo.data_class.__name__)
+    assert algo.gen_func == getattr(mod, algo.gen_func.__name__)
+
+
+def test_globally_unique_executable_family():
+    # Each entry should have a unique executable family
+    fams = [algo.executable_family for algo in BENCHMARKS]
+    assert len(set(fams)) == len(fams)
+
+
+def test_globally_unique_config_full_name():
+    full_names = [config.full_name for algo, config in get_all_algo_configs()]
+    assert len(set(full_names)) == len(full_names)
+
+
+@pytest.mark.parametrize('algo_config', get_all_algo_configs())
+def test_gen_script(algo_config):
+    algo, config = algo_config
+
+    # Make sure it's formulaic
+    assert config.gen_script == f'gen-{config.short_name}.py'
+
+    # Make sure it exists
+    gen_script_path = (f"{RECIRQ_DIR}/{algo.domain.replace('.', '/')}/"
+                       f"{algo.name.replace('.', '/')}/{config.gen_script}")
+    assert os.path.exists(gen_script_path)