Merge branch 'main' into more-hash-tests

Author: JelleZijlstra

.github/workflows/conformance.yml

@@ -0,0 +1,42 @@
+name: Conformance
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  conformance:
+    name: Run conformance suite
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install uv
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install uv
+
+      - name: Run conformance suite
+        working-directory: conformance
+        run: |
+          uv sync --python 3.12 --frozen
+          uv run --python 3.12 --frozen python src/main.py
+
+      - name: Assert conformance results are up to date
+        run: |
+          if [ -n "$(git status --porcelain -- conformance/results)" ]; then
+            git status --short conformance/results
+            git diff -- conformance/results
+            echo "Conformance results are out of date. Run conformance/src/main.py and commit updated results."
+            exit 1
+          fi

conformance/README.md

@@ -70,9 +70,9 @@ by the scoring system.
 
 To run the conformance test suite:
 * Clone the https://github.com/python/typing repo.
-* Create and activate a Python 3.12 virtual environment.
-* Switch to the `conformance` subdirectory and install all dependencies (`pip install -r requirements.txt`).
-* Switch to the `src` subdirectory and run `python main.py`.
+* Install [uv](https://docs.astral.sh/uv/) and ensure Python 3.12 is available.
+* Switch to the `conformance` subdirectory and install locked dependencies (`uv sync --python 3.12 --frozen`).
+* Run the conformance tool (`uv run --python 3.12 --frozen python src/main.py`).
 
 Note that some type checkers may not run on some platforms. If a type checker fails to install, tests will be skipped for that type checker.
 
@@ -92,7 +92,22 @@ If a test is updated (augmented or fixed), the process is similar to when adding
 
 ## Updating a Type Checker
 
-If a new version of a type checker is released, re-run the test tool with the new version. If the type checker output has changed for any test cases, the tool will supply the old and new outputs. Examine these to determine whether the conformance status has changed. Once the conformance status has been updated, re-run the test tool again to regenerate the summary report.
+Type checker versions are locked in `uv.lock`.
+
+To bump all supported checkers to their latest released versions:
+
+```bash
+python scripts/bump_type_checkers.py
+```
+
+After bumping, install the new lockfile and rerun conformance:
+
+```bash
+uv sync --python 3.12 --frozen
+uv run --python 3.12 --frozen python src/main.py
+```
+
+If checker output changes for any test cases, examine those deltas to determine whether the conformance status has changed. Once the conformance status has been updated, rerun the tool to regenerate the summary report.
 
 ## Automated Conformance Checking
 

conformance/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "typing-conformance"
+version = "0.1.0"
+requires-python = "==3.12.*"
+dependencies = [
+    "mypy",
+    "pyrefly",
+    "pyright",
+    "tomli",
+    "tomlkit",
+    "zuban",
+]
+
+[tool.uv]
+package = false

conformance/requirements.txt

@@ -1,6 +1,5 @@
 conformant = "Pass"
 output = """
-qualifiers_final_decorator.py:8:6 - warning: Import "_qualifiers_final_decorator" could not be resolved from source (reportMissingModuleSource)
 qualifiers_final_decorator.py:21:16 - error: Base class "Base1" is marked final and cannot be subclassed (reportGeneralTypeIssues)
 qualifiers_final_decorator.py:56:9 - error: Method "method1" cannot override final method defined in class "Base2" (reportIncompatibleMethodOverride)
 qualifiers_final_decorator.py:60:9 - error: Method "method2" cannot override final method defined in class "Base2" (reportIncompatibleMethodOverride)

conformance/results/pyright/qualifiers_final_decorator.toml

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""
+Update the conformance checker versions in uv.lock to the latest releases.
+"""
+
+from pathlib import Path
+from subprocess import CalledProcessError, run
+
+
+TYPE_CHECKERS = ("mypy", "pyright", "zuban", "pyrefly")
+
+
+def main() -> int:
+    root_dir = Path(__file__).resolve().parents[1]
+    lock_command = ["uv", "lock", "--python", "3.12"]
+    for checker in TYPE_CHECKERS:
+        lock_command.extend(["--upgrade-package", checker])
+
+    try:
+        print("+", " ".join(lock_command))
+        run(lock_command, cwd=root_dir, check=True)
+    except CalledProcessError as exc:
+        print(f"Failed with exit code {exc.returncode}")
+        return exc.returncode
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

conformance/scripts/bump_type_checkers.py

@@ -5,8 +5,6 @@
 from abc import ABC, abstractmethod
 import json
 from pathlib import Path
-import os
-import re
 import shutil
 from subprocess import PIPE, CalledProcessError, run
 import sys
@@ -25,8 +23,8 @@ def name(self) -> str:
     @abstractmethod
     def install(self) -> bool:
         """
-        Ensures that the latest version of the type checker is installed.
-        Returns False if installation fails.
+        Ensures that the type checker is available in the current environment.
+        Returns False if it cannot be executed.
         """
         raise NotImplementedError
 
@@ -67,29 +65,24 @@ def install(self) -> bool:
             pass
 
         try:
-            # Uninstall any existing version if present.
-            run(
-                [sys.executable, "-m", "pip", "uninstall", "mypy", "-y"],
-                check=True,
-            )
-
-            # Install the latest version.
-            run(
-                [sys.executable, "-m", "pip", "install", "mypy"],
-                check=True,
-            )
-
-            # Run "mypy --version" to ensure that it's installed and to work
+            # Run "mypy --version" to ensure that it's available and to work
             # around timing issues caused by malware scanners on some systems.
             self.get_version()
-
             return True
-        except CalledProcessError:
-            print("Unable to install mypy")
+        except (CalledProcessError, FileNotFoundError):
+            print(
+                "Unable to run mypy. Install conformance dependencies with "
+                "'uv sync --frozen' from the conformance directory."
+            )
             return False
 
     def get_version(self) -> str:
-        proc = run([sys.executable, "-m", "mypy", "--version"], stdout=PIPE, text=True)
+        proc = run(
+            [sys.executable, "-m", "mypy", "--version"],
+            check=True,
+            stdout=PIPE,
+            text=True,
+        )
         version = proc.stdout.strip()
 
         # Remove the " (compiled)" if it's present.
@@ -137,29 +130,23 @@ def name(self) -> str:
 
     def install(self) -> bool:
         try:
-            # Uninstall any old version if present.
-            run(
-                [sys.executable, "-m", "pip", "uninstall", "pyright", "-y"],
-                check=True,
-            )
-
-            # Install the latest version.
-            run(
-                [sys.executable, "-m", "pip", "install", "pyright"],
-                check=True,
-            )
-
             # Force the Python wrapper to install node if needed
-            # and download the latest version of pyright.
+            # and use the locked version of pyright.
             self.get_version()
             return True
-        except CalledProcessError:
-            print("Unable to install pyright")
+        except (CalledProcessError, FileNotFoundError):
+            print(
+                "Unable to run pyright. Install conformance dependencies with "
+                "'uv sync --frozen' from the conformance directory."
+            )
             return False
 
     def get_version(self) -> str:
         proc = run(
-            [sys.executable, "-m", "pyright", "--version"], stdout=PIPE, text=True
+            [sys.executable, "-m", "pyright", "--version"],
+            check=True,
+            stdout=PIPE,
+            text=True,
         )
         return proc.stdout.strip()
 
@@ -208,24 +195,17 @@ def name(self) -> str:
 
     def install(self) -> bool:
         try:
-            # Uninstall any existing version if present.
-            run(
-                [sys.executable, "-m", "pip", "uninstall", "zuban", "-y"],
-                check=True,
-            )
-
-            # Install the latest version.
-            run(
-                [sys.executable, "-m", "pip", "install", "zuban"],
-                check=True,
-            )
+            self.get_version()
             return True
-        except CalledProcessError:
-            print("Unable to install zuban")
+        except (CalledProcessError, FileNotFoundError):
+            print(
+                "Unable to run zuban. Install conformance dependencies with "
+                "'uv sync --frozen' from the conformance directory."
+            )
             return False
 
     def get_version(self) -> str:
-        proc = run(["zuban", "--version"], stdout=PIPE, text=True)
+        proc = run(["zuban", "--version"], check=True, stdout=PIPE, text=True)
         return proc.stdout.strip()
 
     def run_tests(self, test_files: Sequence[str]) -> dict[str, str]:
@@ -268,23 +248,17 @@ def name(self) -> str:
 
     def install(self) -> bool:
         try:
-            # Uninstall any existing version if present.
-            run(
-                [sys.executable, "-m", "pip", "uninstall", "pyrefly", "-y"],
-                check=True,
-            )
-            # Install the latest version.
-            run(
-                [sys.executable, "-m", "pip", "install", "pyrefly"],
-                check=True,
-            )
+            self.get_version()
             return True
-        except CalledProcessError:
-            print("Unable to install pyrefly")
+        except (CalledProcessError, FileNotFoundError):
+            print(
+                "Unable to run pyrefly. Install conformance dependencies with "
+                "'uv sync --frozen' from the conformance directory."
+            )
             return False
 
     def get_version(self) -> str:
-        proc = run(["pyrefly", "--version"], stdout=PIPE, text=True)
+        proc = run(["pyrefly", "--version"], check=True, stdout=PIPE, text=True)
         version = proc.stdout.strip()
         return version
 

conformance/src/type_checker.py

@@ -42,7 +42,7 @@ def callback2(a: int, d: str) -> tuple[str]:
 assert_type(func2(callback2), tuple[str])
 
 
-def func3(*args: * tuple[int, *Ts, T]) -> tuple[T, *Ts]:
+def func3(*args: *tuple[int, *Ts, T]) -> tuple[T, *Ts]:
     raise NotImplementedError