Merge branch 'main' into issue-102541

Author: arhadthedev

.devcontainer/Dockerfile

@@ -0,0 +1,24 @@
+FROM docker.io/library/fedora:37
+
+ENV CC=clang
+
+ENV WASI_SDK_VERSION=19
+ENV WASI_SDK_PATH=/opt/wasi-sdk
+
+ENV WASMTIME_HOME=/opt/wasmtime
+ENV WASMTIME_VERSION=7.0.0
+ENV WASMTIME_CPU_ARCH=x86_64
+
+RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \
+    dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \
+    dnf -y clean all
+
+RUN mkdir ${WASI_SDK_PATH} && \
+    curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \
+    tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip
+
+RUN mkdir --parents ${WASMTIME_HOME} && \
+    curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \
+    xz --decompress | \
+    tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \
+    ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin

.devcontainer/devcontainer.json

@@ -0,0 +1,81 @@
+{
+    "build": {
+        "dockerfile": "Dockerfile"
+    },
+    "onCreateCommand": [
+        // Install common tooling.
+        "dnf",
+        "install",
+        "-y",
+        "which",
+        "zsh",
+        "fish"
+    ],
+    "updateContentCommand": {
+        // Using the shell for `nproc` usage.
+        "python": "./configure --config-cache --with-pydebug && make -s -j `nproc`",
+        "docs": [
+            "make",
+            "--directory",
+            "Doc",
+            "venv",
+            "html"
+        ]
+    },
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                // Highlighting for Parser/Python.asdl.
+                "brettcannon.zephyr-asdl",
+                // Highlighting for configure.ac.
+                "maelvalais.autoconf",
+                // C auto-complete.
+                "ms-vscode.cpptools",
+                // To view built docs.
+                "ms-vscode.live-server"
+                // https://github.com/microsoft/vscode-python/issues/18073
+                // "ms-python.python"
+            ],
+            "settings": {
+                "C_Cpp.default.compilerPath": "/usr/bin/clang",
+                "C_Cpp.default.cStandard": "c11",
+                "C_Cpp.default.defines": [
+                    "CONFIG_64",
+                    "Py_BUILD_CORE"
+                ],
+                "C_Cpp.default.includePath": [
+                    "${workspaceFolder}/*",
+                    "${workspaceFolder}/Include/**"
+                ],
+                // https://github.com/microsoft/vscode-cpptools/issues/10732
+                "C_Cpp.errorSquiggles": "disabled",
+                "editor.insertSpaces": true,
+                "editor.rulers": [
+                    80
+                ],
+                "editor.tabSize": 4,
+                "editor.trimAutoWhitespace": true,
+                "files.associations": {
+                    "*.h": "c"
+                },
+                "files.encoding": "utf8",
+                "files.eol": "\n",
+                "files.insertFinalNewline": true,
+                "files.trimTrailingWhitespace": true,
+                "python.analysis.diagnosticSeverityOverrides": {
+                    // Complains about shadowing the stdlib w/ the stdlib.
+                    "reportShadowedImports": "none",
+                    // Doesn't like _frozen_importlib.
+                    "reportMissingImports": "none"
+                },
+                "python.analysis.extraPaths": [
+                    "Lib"
+                ],
+                "python.defaultInterpreterPath": "./python",
+                "[restructuredtext]": {
+                    "editor.tabSize": 3
+                }
+            }
+        }
+    }
+}

.gitattributes

@@ -32,6 +32,10 @@ Lib/test/test_importlib/resources/data01/*           noeol
 Lib/test/test_importlib/resources/namespacedata01/*  noeol
 Lib/test/xmltestdata/*                     noeol
 
+# Shell scripts should have LF even on Windows because of Cygwin
+Lib/venv/scripts/common/activate text eol=lf
+Lib/venv/scripts/posix/* text eol=lf
+
 # CRLF files
 [attr]dos text eol=crlf
 

.github/CODEOWNERS

@@ -5,13 +5,16 @@
 # https://git-scm.com/docs/gitignore#_pattern_format
 
 # GitHub
-.github/**                    @ezio-melotti
+.github/**                    @ezio-melotti @hugovk
+
+# pre-commit
+.pre-commit-config.yaml       @hugovk @AlexWaygood
 
 # Build system
 configure*                    @erlend-aasland @corona10
 
 # asyncio
-**/*asyncio*                  @1st1 @asvetlov @gvanrossum @kumaraditya303
+**/*asyncio*                  @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc
 
 # Core
 **/*context*                  @1st1
@@ -25,6 +28,8 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval.c                @markshannon
 Python/compile.c              @markshannon @iritkatriel
+Python/assemble.c             @markshannon @iritkatriel
+Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_peepholer.py    @brandtbucher
@@ -61,11 +66,7 @@ Python/traceback.c            @iritkatriel
 /Tools/build/parse_html5_entities.py   @ezio-melotti
 
 # Import (including importlib).
-# Ignoring importlib.h so as to not get flagged on
-# all pull requests that change the emitted
-# bytecode.
-**/*import*.c                 @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
-**/*import*.py                @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
+**/*import*                   @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw
 **/*importlib/resources/*      @jaraco @warsaw @FFY00
 **/importlib/metadata/*       @jaraco @warsaw
 

.github/workflows/build.yml

@@ -33,8 +33,10 @@ jobs:
   check_source:
     name: 'Check for source changes'
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -60,9 +62,21 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
@@ -111,10 +125,14 @@ jobs:
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
+      - name: Check for unsupported C global variables
+        if: github.event_name == 'pull_request'  # $GITHUB_EVENT_NAME
+        run: make check-c-globals
 
   build_win32:
     name: 'Windows (x86)'
     runs-on: windows-latest
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
@@ -123,7 +141,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build CPython
       run: .\PCbuild\build.bat -e -d -p Win32
-      timeout-minutes: 30
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
@@ -132,6 +149,7 @@ jobs:
   build_win_amd64:
     name: 'Windows (x64)'
     runs-on: windows-latest
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
@@ -142,7 +160,6 @@ jobs:
       run: echo "::add-matcher::.github/problem-matchers/msvc.json"
     - name: Build CPython
       run: .\PCbuild\build.bat -e -d -p x64
-      timeout-minutes: 30
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
@@ -151,6 +168,7 @@ jobs:
   build_macos:
     name: 'macOS'
     runs-on: macos-latest
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
@@ -181,6 +199,7 @@ jobs:
   build_ubuntu:
     name: 'Ubuntu'
     runs-on: ubuntu-20.04
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
@@ -238,6 +257,7 @@ jobs:
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
     runs-on: ubuntu-20.04
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
@@ -283,10 +303,97 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -o \
+          -j4 \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'
     runs-on: ubuntu-20.04
+    timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
@@ -299,6 +406,10 @@ jobs:
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Set up GCC-10 for ASAN
+      uses: egor-tensin/setup-gcc@v1
+      with:
+        version: 10
     - name: Configure OpenSSL env vars
       run: |
         echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV

.github/workflows/build_msi.yml

@@ -26,6 +26,7 @@ jobs:
   build:
     name: Windows Installer
     runs-on: windows-latest
+    timeout-minutes: 60
     strategy:
       matrix:
         type: [x86, x64, arm64]