Algo Benchmark Library card catalog (#235)

Author: mpharrigan

.github/workflows/pythonpackage.yml

@@ -51,8 +51,8 @@ jobs:
         run: |
           pip install pytest
           # RECIRQ_CHESS_TEST_SEED: make quantum_chess tests determnistic
-          # RECIRQ_IGNORE_PYTKET: for 'next' cirq version, skip pytket tests
-          RECIRQ_CHESS_TEST_SEED=789 RECIRQ_IGNORE_PYTKET=y pytest -v
+          # RECIRQ_IMPORT_FAILSAFE: skip tests on unsupported Cirq configurations
+          RECIRQ_CHESS_TEST_SEED=789 RECIRQ_IMPORT_FAILSAFE=y pytest -v
   nbformat:
     name: Notebook formatting
     runs-on: ubuntu-latest

recirq/algorithmic_benchmark_library.py

@@ -0,0 +1,99 @@
+import dataclasses
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Type, Callable, List, Tuple, Any
+
+import pandas as pd
+
+try:
+    from cirq_google.workflow import ExecutableSpec, QuantumExecutableGroup
+
+    workflow = True
+except ImportError as e:
+    import os
+
+    if 'RECIRQ_IMPORT_FAILSAFE' in os.environ:
+        workflow = False
+    else:
+        raise ImportError(f"This functionality requires a pre-release version of Cirq: {e}")
+
+
+@dataclass
+class AlgorithmicBenchmark:
+    """A record of relevant elements that comprise a benchmark based on a quantum computing
+    algorithm of interest.
+
+    Args:
+        domain: The problem domain represented as the corresponding high-level ReCirq module.
+            This must be a valid module name beginning with "recirq.".
+        name: The benchmark name. Must be unique within the domain.
+        executable_family: A globally unique identifier for this AlgorithmicBenchmark.
+            This should match up with this Benchmark's `spec_class.executable_family`.
+            The executable family is the fully-qualified leaf-module where the code
+            for this AlgorithmicBenchmark lives.
+        spec_class: The ExecutableSpec subclass for this AlgorithmicBenchmark.
+        data_class: The class which can contain ETL-ed data for this AlgorithmicBenchmark.
+        executable_generator_func: The function that returns a QuantumExecutableGroup for a
+            given Config.
+        configs: A list of available `BenchmarkConfig` for this benchmark.
+    """
+
+    domain: str
+    name: str
+    executable_family: str
+    spec_class: Type['ExecutableSpec']
+    data_class: Type
+    executable_generator_func: Callable[[Any], 'QuantumExecutableGroup']
+    configs: List['BenchmarkConfig']
+
+    def as_strings(self):
+        """Get values of this class as strings suitable for printing."""
+        ret = {k: str(v) for k, v in dataclasses.asdict(self).items()}
+        ret['spec_class'] = self.spec_class.__name__
+        ret['data_class'] = self.data_class.__name__
+        ret['executable_generator_func'] = self.executable_generator_func.__name__
+        return ret
+
+
+@dataclass
+class BenchmarkConfig:
+    """A particular configuration of an AlgorithmicBenchmark
+
+    Args:
+        short_name: The short name for this config. Unique within an AlgorithmicBenchmark.
+        full_name: A globally unique name for this config.
+        gen_script: The script filename that generates the QuantumExecutableGroup for this Config.
+            Should begin with prefix "gen-".
+        run_scripts: A list of script filenames that execute (or can execute) this config.
+    """
+
+    short_name: str
+    full_name: str
+    gen_script: str
+    run_scripts: List[str]
+
+
+BENCHMARKS = [
+]
+
+
+@lru_cache()
+def get_all_algo_configs() -> List[Tuple[AlgorithmicBenchmark, BenchmarkConfig]]:
+    """Return a tuple of (AlgorithmsBenchmark, BenchmarkConfig) for each BenchmarkConfig in
+    `BENCHMARKS`.
+
+    This "flattens" the list of BenchmarkConfig from their nested structure in `BENCHMARKS`.
+    """
+    ret = []
+    for algo in BENCHMARKS:
+        for config in algo.configs:
+            ret.append((algo, config))
+    return ret
+
+
+def print_table_of_benchmarks():
+    """Print the AlgorithmicBenchmarks in a tabular form."""
+    df = pd.DataFrame([algo.as_strings() for algo in BENCHMARKS]).set_index('executable_family')
+    pd.set_option('display.width', None)
+    pd.set_option('display.max_colwidth', None)
+    print(df.transpose())

recirq/algorithmic_benchmark_library_test.py

@@ -0,0 +1,80 @@
+import os
+from importlib import import_module
+from types import ModuleType
+
+import pytest
+
+from recirq.algorithmic_benchmark_library import BENCHMARKS, get_all_algo_configs, workflow
+
+RECIRQ_DIR = os.path.abspath(os.path.dirname(__file__) + '/../')
+
+if not workflow:
+    pytestmark = pytest.mark.skip('algorithmic_benchmark_library requires pre-release of Cirq.')
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_domain(algo):
+    # By convention, the domain should be a recirq module.
+    assert algo.domain.startswith('recirq.'), 'domain should be a recirq module.'
+    mod = import_module(algo.domain)
+    assert isinstance(mod, ModuleType), 'domain should be a recirq module.'
+
+
+def test_benchmark_name_unique_in_domain():
+    # In a given domain, all benchmark names should be unique
+    pairs = [(algo.domain, algo.name) for algo in BENCHMARKS]
+    assert len(set(pairs)) == len(pairs)
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_executable_family_is_formulaic(algo):
+    # Check consistency in the AlgorithmicBenchmark dataclass:
+    assert algo.executable_family == algo.spec_class.executable_family, \
+        "benchmark's executable_family should match that of the spec_class"
+
+    # By convention, we set this to be the module name. By further convention,
+    # {algo.domain}.{algo.name} should be the module name.
+    assert algo.executable_family == f'{algo.domain}.{algo.name}', \
+        "The executable family should be set to the benchmarks's domain.name"
+
+    # Check the convention that it should give a module
+    mod = import_module(algo.executable_family)
+    assert isinstance(mod, ModuleType), \
+        "The executable family should specify an importable module."
+
+
+@pytest.mark.parametrize('algo', BENCHMARKS)
+def test_classes_and_funcs(algo):
+    # The various class objects should exist in the module
+    mod = import_module(algo.executable_family)
+    assert algo.spec_class == getattr(mod, algo.spec_class.__name__), \
+        "The spec_class must exist in the benchmark's module"
+    assert algo.data_class == getattr(mod, algo.data_class.__name__), \
+        "The data_class must exist in the benchmark's module"
+    assert algo.executable_generator_func == getattr(mod, algo.executable_generator_func.__name__), \
+        "the executable_generator_func must exist in the benchmark's module"
+
+
+def test_globally_unique_executable_family():
+    # Each entry should have a unique executable family
+    fams = [algo.executable_family for algo in BENCHMARKS]
+    assert len(set(fams)) == len(fams)
+
+
+def test_globally_unique_config_full_name():
+    full_names = [config.full_name for algo, config in get_all_algo_configs()]
+    assert len(set(full_names)) == len(full_names)
+
+
+@pytest.mark.parametrize('algo_config', get_all_algo_configs())
+def test_gen_script(algo_config):
+    algo, config = algo_config
+
+    # Make sure it's formulaic
+    assert config.gen_script == f'gen-{config.short_name}.py', \
+        "The gen_script should be of the form 'gen-{short_name}'"
+
+    # Make sure it exists
+    gen_script_path = (f"{RECIRQ_DIR}/{algo.domain.replace('.', '/')}/"
+                       f"{algo.name.replace('.', '/')}/{config.gen_script}")
+    assert os.path.exists(gen_script_path)

recirq/qaoa/placement.py

@@ -11,7 +11,7 @@
 import cirq_google as cg
 
 try:
-    # Set the 'RECIRQ_IGNORE_PYTKET' environment variable to treat PyTket as an optional
+    # Set the 'RECIRQ_IMPORT_FAILSAFE' environment variable to treat PyTket as an optional
     # dependency. We do this for CI testing against the next, pre-release Cirq version.
     import pytket
     import pytket.extensions.cirq
@@ -20,7 +20,7 @@
     from pytket.predicates import CompilationUnit, ConnectivityPredicate
     from pytket.routing import GraphPlacement
 except ImportError as e:
-    if 'RECIRQ_IGNORE_PYTKET' in os.environ:
+    if 'RECIRQ_IMPORT_FAILSAFE' in os.environ:
         pytket = NotImplemented
     else:
         raise e