Add the ability to build parsing with mypyc

If called with `PARSING_USE_MYPYC=1` environment variable set,
`setup.py` will now produce a compiled version of parser table
generation code, which speeds it up by roughly a factor of two,
reducing total EdgeDB parser table generation time to about
13 seconds.

Author: elprans

.github/workflows/test.yml

@@ -8,14 +8,15 @@ on:
       - "releases/*"
   pull_request:
     branches:
-      - '*'
+      - "*"
 
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.7, 3.8, 3.9, 3.10.0-rc.1]
+        build: ["mypyc", "pure-python"]
 
     env:
       PIP_DISABLE_PIP_VERSION_CHECK: 1
@@ -27,6 +28,17 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test
+        env:
+          PARSING_USE_MYPYC: "${{ matrix.build == 'mypyc' && '1' || '0' }}"
         run: |
           python -m pip install -e .[test]
           python -m unittest -v parsing.tests.suite
+
+  # This job exists solely to act as the test job aggregate to be
+  # targeted by branch policies.
+  regression-tests:
+    name: "Regression Tests"
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo OK

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 *.pyo
+*.so
 __pycache__/
 /build
 /dist
@@ -8,3 +9,4 @@ __pycache__/
 /.eggs
 /.vscode
 /.mypy_cache
+/.eggs

setup.py

@@ -1,15 +1,83 @@
+import os
+import sys
+
+from setuptools import extension as setuptools_ext
 from setuptools import setup
+from setuptools.command import build_ext as setuptools_build_ext
 
 
 extra = {}
 
 f = open("README.rst", "r")
 try:
     extra["long_description"] = f.read()
+    extra["long_description_content_type"] = "text/x-rst"
 finally:
     f.close()
 
 
+USE_MYPYC = False
+MYPY_DEPENDENCY = "mypy>=0.910"
+setup_requires = []
+ext_modules = []
+
+if (
+    os.environ.get("PARSING_USE_MYPYC", None) in {"true", "1", "on"}
+    or "--use-mypyc" in sys.argv
+):
+    setup_requires.append(MYPY_DEPENDENCY)
+    # Fool setuptools into calling build_ext.  The actual list of
+    # extensions would get replaced by mypycify.
+    ext_modules.append(
+        setuptools_ext.Extension("parsing.foo", ["parsing/foo.c"])
+    )
+    USE_MYPYC = True
+
+
+class build_ext(setuptools_build_ext.build_ext):  # type: ignore
+    def finalize_options(self) -> None:
+        # finalize_options() may be called multiple times on the
+        # same command object, so make sure not to override previously
+        # set options.
+        if getattr(self, "_initialized", False):
+            return
+
+        if USE_MYPYC:
+            import pkg_resources
+
+            # Double check Cython presence in case setup_requires
+            # didn't go into effect (most likely because someone
+            # imported Cython before setup_requires injected the
+            # correct egg into sys.path.
+            try:
+                import mypy.version
+                from mypyc.build import mypycify
+            except ImportError:
+                raise RuntimeError(
+                    "please install {} to compile parsing from source".format(
+                        MYPY_DEPENDENCY
+                    )
+                )
+
+            mypy_dep = pkg_resources.Requirement.parse(MYPY_DEPENDENCY)
+            if mypy.version.__version__ not in mypy_dep:
+                raise RuntimeError(
+                    "parsing requires {}, got mypy=={}".format(
+                        MYPY_DEPENDENCY, mypy.version.__version__
+                    )
+                )
+
+            self.distribution.ext_modules = mypycify(
+                [
+                    "parsing/automaton.py",
+                    "parsing/grammar.py",
+                ],
+                separate=True,
+            )
+
+        super(build_ext, self).finalize_options()
+
+
 setup(
     name="parsing",
     version="2.0.0.dev0",
@@ -31,11 +99,14 @@
     ],
     packages=["parsing", "parsing.tests", "parsing.tests.specs"],
     package_data={"parsing": ["py.typed"]},
+    setup_requires=setup_requires,
+    ext_modules=ext_modules,
     extras_require={
         "test": [
             "flake8",
             MYPY_DEPENDENCY,
         ]
     },
+    cmdclass={"build_ext": build_ext},
     **extra,
 )