diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 05f20e12f4653d..665befea8787c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -260,8 +260,8 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' + build-ubuntu-ssltests: + name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context @@ -269,75 +269,19 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] + include: + - { os: ubuntu-24.04, ssl: openssl, ssl_ver: 3.0.16 } + - { os: ubuntu-24.04, ssl: openssl, ssl_ver: 3.1.8 } + - { os: ubuntu-24.04, ssl: openssl, ssl_ver: 3.2.4 } + - { os: ubuntu-24.04, ssl: openssl, ssl_ver: 3.3.3 } + - { os: ubuntu-24.04, ssl: openssl, ssl_ver: 3.4.1 } + - { os: ubuntu-24.04, ssl: awslc, ssl_ver: 1.55.0 } # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] - env: - AWSLC_VER: ${{ matrix.awslc_ver}} + SSL_VER: ${{ matrix.ssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + SSL_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssl }}/${{ matrix.ssl_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssl }}/${{ matrix.ssl_ver }}/lib steps: - uses: actions/checkout@v4 with: @@ -356,22 +300,18 @@ jobs: - name: Configure SSL lib env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc + echo "SSL_DIR=${GITHUB_WORKSPACE}/multissl/${{ matrix.ssl }}/${SSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/${{ matrix.ssl }}/${SSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore SSL build' + id: cache-ssl uses: actions/cache@v4 with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' + path: ./multissl/${{ env.SSL }}/${{ env.SSL_VER }} + key: ${{ matrix.os }}-multissl-${{ env.SSL }}-${{ env.SSL_VER }} + - name: Install SSL + if: steps.cache-ssl.outputs.cache-hit != 'true' run: | - python3 Tools/ssl/multissltests.py \ - --steps=library \ - --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ - --system Linux + python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --system Linux --ssl ${{ matrix.ssl }} --ssl-versions ${{ matrix.ssl_ver }} - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -381,18 +321,18 @@ jobs: save: false - name: Configure CPython run: | - ./configure CFLAGS="-fdiagnostics-format=json" \ - --config-cache \ - --enable-slower-safety \ - --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ - --with-builtin-hashlib-hashes=blake2 \ - --with-ssl-default-suites=openssl + CMD=(./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$SSL_DIR") + if [ "${{ matrix.ssl }}" = "openssl" ]; then + "${CMD[@]}" + else + "${CMD[@]}" --with-builtin-hashlib-hashes=blake2 --with-ssl-default-suites=openssl + fi - name: Build CPython - run: make -j + run: make -j4 - name: Display build info run: make pythoninfo - name: Verify python is linked to AWS-LC + if: matrix.ssl == 'aws-lc' run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC - name: SSL tests run: ./python Lib/test/ssltests.py @@ -435,7 +375,7 @@ jobs: key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl 'openssl' --ssl-versions "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -567,7 +507,7 @@ jobs: key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl 'openssl' --ssl-versions "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -703,8 +643,7 @@ jobs: - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl + - build-ubuntu-ssltests - build-wasi - test-hypothesis - build-asan @@ -719,8 +658,7 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, + build-ubuntu-ssltests, test-hypothesis, cifuzz, allowed-skips: >- @@ -738,8 +676,7 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, + build-ubuntu-ssltests, build-wasi, test-hypothesis, build-asan, diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a72e..607e7949161812 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -60,7 +60,7 @@ jobs: key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --ssl openssl --ssl-versions "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index f4c8fde8346fd9..2c8264dd09076b 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -18,12 +18,15 @@ search paths for header files and shared libraries. It's known to work on Linux with GCC and clang. -Please keep this script compatible with Python 2.7, and 3.4 to 3.7. +Please keep this script compatible with all currently-maintained Python +versions. (c) 2013-2017 Christian Heimes """ from __future__ import print_function +from abc import abstractmethod +from abc import ABCMeta import argparse from datetime import datetime import logging @@ -42,29 +45,6 @@ log = logging.getLogger("multissl") -OPENSSL_OLD_VERSIONS = [ - "1.1.1w", -] - -OPENSSL_RECENT_VERSIONS = [ - "3.0.16", - "3.1.8", - "3.2.4", - "3.3.3", - "3.4.1", - # See make_ssl_data.py for notes on adding a new version. -] - -LIBRESSL_OLD_VERSIONS = [ -] - -LIBRESSL_RECENT_VERSIONS = [ -] - -AWSLC_RECENT_VERSIONS = [ - "1.55.0", -] - # store files in ../multissl HERE = os.path.dirname(os.path.abspath(__file__)) PYTHONROOT = os.path.abspath(os.path.join(HERE, '..', '..')) @@ -78,6 +58,7 @@ "versions." ), ) +parser.color = True parser.add_argument( '--debug', action='store_true', @@ -86,33 +67,19 @@ parser.add_argument( '--disable-ancient', action='store_true', - help="Don't test OpenSSL and LibreSSL versions without upstream support", + help="Don't test SSL versions without upstream support", ) parser.add_argument( - '--openssl', - nargs='+', - default=(), - help=( - "OpenSSL versions, defaults to '{}' (ancient: '{}') if no " - "OpenSSL and LibreSSL versions are given." - ).format(OPENSSL_RECENT_VERSIONS, OPENSSL_OLD_VERSIONS) -) -parser.add_argument( - '--libressl', - nargs='+', - default=(), - help=( - "LibreSSL versions, defaults to '{}' (ancient: '{}') if no " - "OpenSSL and LibreSSL versions are given." - ).format(LIBRESSL_RECENT_VERSIONS, LIBRESSL_OLD_VERSIONS) + '--ssl', + choices=['openssl', 'awslc', 'libressl'], + default=None, + help="Which SSL lib to test. If not specified, all are tested.", ) parser.add_argument( - '--awslc', + '--ssl-versions', nargs='+', - default=(), - help=( - "AWS-LC versions, defaults to '{}' if no crypto library versions are given." - ).format(AWSLC_RECENT_VERSIONS) + default=None, + help="SSL lib version(s), default depends on value passed to --ssl", ) parser.add_argument( '--tests', @@ -160,30 +127,55 @@ ) -class AbstractBuilder(object): - library = None - url_templates = None - src_template = None - build_template = None +class AbstractBuilder(object, metaclass=ABCMeta): depend_target = None install_target = 'install' if hasattr(os, 'process_cpu_count'): jobs = os.process_cpu_count() else: jobs = os.cpu_count() - module_files = ( os.path.join(PYTHONROOT, "Modules/_ssl.c"), os.path.join(PYTHONROOT, "Modules/_hashopenssl.c"), ) module_libs = ("_ssl", "_hashlib") + @property + @abstractmethod + def library(self=None): + pass + + @property + @abstractmethod + def url_templates(self=None): + pass + + @property + @abstractmethod + def src_template(self=None): + pass + + @property + @abstractmethod + def build_template(self=None): + pass + + @property + @abstractmethod + def recent_versions(): + pass + + @property + def old_versions(): + return [] + def __init__(self, version, args): self.version = version self.args = args + libdir = self.library.lower().replace("-", "") # installation directory self.install_dir = os.path.join( - os.path.join(args.base_directory, self.library.lower()), version + os.path.join(args.base_directory, libdir), version ) # source file self.src_dir = os.path.join(args.base_directory, 'src') @@ -410,18 +402,45 @@ def run_python_tests(self, tests, network=True): class BuildOpenSSL(AbstractBuilder): - library = "OpenSSL" - url_templates = ( - "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz", - "https://www.openssl.org/source/openssl-{v}.tar.gz", - "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz" - ) - src_template = "openssl-{}.tar.gz" - build_template = "openssl-{}" # only install software, skip docs install_target = 'install_sw' depend_target = 'depend' + @property + def library(self=None): + return "OpenSSL" + + @property + def url_templates(self=None): + return ( + "https://github.com/openssl/openssl/releases/download/openssl-{v}/openssl-{v}.tar.gz", + "https://www.openssl.org/source/openssl-{v}.tar.gz", + "https://www.openssl.org/source/old/{s}/openssl-{v}.tar.gz", + ) + + @property + def src_template(self=None): + return "openssl-{}.tar.gz" + + @property + def build_template(self=None): + return "openssl-{}" + + @property + def recent_versions(): + return [ + "3.0.16", + "3.1.8", + "3.2.4", + "3.3.3", + "3.4.1", + # See make_ssl_data.py for notes on adding a new version. + ] + + @property + def old_versions(): + return [ "1.1.1w" ] + def _post_install(self): if self.version.startswith("3."): self._post_install_3xx() @@ -457,21 +476,53 @@ def short_version(self): class BuildLibreSSL(AbstractBuilder): - library = "LibreSSL" - url_templates = ( - "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{v}.tar.gz", - ) - src_template = "libressl-{}.tar.gz" - build_template = "libressl-{}" + @property + def library(self=None): + return "LibreSSL" + + @property + def url_templates(self=None): + return ( + "https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-{v}.tar.gz", + ) + + @property + def src_template(self=None): + return "libressl-{}.tar.gz" + + @property + def build_template(self=None): + "libressl-{}" + + @property + def recent_versions(): + return [] class BuildAWSLC(AbstractBuilder): - library = "AWS-LC" - url_templates = ( - "https://github.com/aws/aws-lc/archive/refs/tags/v{v}.tar.gz", - ) - src_template = "aws-lc-{}.tar.gz" - build_template = "aws-lc-{}" + @property + def library(self=None): + return "AWS-LC" + + @property + def url_templates(self=None): + return ( + "https://github.com/aws/aws-lc/archive/refs/tags/v{v}.tar.gz", + ) + + @property + def src_template(self=None): + return "aws-lc-{}.tar.gz" + + @property + def build_template(self=None): + return "aws-lc-{}" + + @property + def recent_versions(): + return [ + "1.55.0", + ] def _build_src(self, config_args=()): cwd = self.build_dir @@ -507,19 +558,6 @@ def configure_make(): def main(): args = parser.parse_args() - if not args.openssl and not args.libressl and not args.awslc: - args.openssl = list(OPENSSL_RECENT_VERSIONS) - args.libressl = list(LIBRESSL_RECENT_VERSIONS) - args.awslc = list(AWSLC_RECENT_VERSIONS) - if not args.disable_ancient: - args.openssl.extend(OPENSSL_OLD_VERSIONS) - args.libressl.extend(LIBRESSL_OLD_VERSIONS) - - logging.basicConfig( - level=logging.DEBUG if args.debug else logging.INFO, - format="*** %(levelname)s %(message)s" - ) - start = datetime.now() if args.steps in {'modules', 'tests'}: @@ -535,17 +573,38 @@ def main(): # check for configure and run make configure_make() - # download and register builder + logging.basicConfig( + level=logging.DEBUG if args.debug else logging.INFO, + format="*** %(levelname)s %(message)s" + ) + + versions = [] + ssl_libs = AbstractBuilder.__subclasses__() + if args.ssl: + lib_name = lambda x: x.library.fget().lower().replace("-", "") + libs = [l for l in ssl_libs if lib_name(l) == args.ssl] + assert len(libs) == 1 + cls = libs.pop() + if args.ssl_versions: + versions += [(cls, v) for v in args.ssl_versions] + else: + versions += [(cls, v) for v in cls.recent_versions.fget()] + if not args.disable_ancient: + versions += [(cls, v) for v in cls.old_versions.fget()] + else: + if args.ssl_versions: + print("ERROR: SSL versions specified without specifying library") + exit(1) + for cls in ssl_libs: + versions += [(cls, v) for v in cls.recent_versions.fget()] + if not args.disable_ancient: + versions += [(cls, v) for v in cls.old_versions.fget()] + builds = [] - for build_class, versions in [ - (BuildOpenSSL, args.openssl), - (BuildLibreSSL, args.libressl), - (BuildAWSLC, args.awslc), - ]: - for version in versions: - build = build_class(version, args) - build.install() - builds.append(build) + for build_class, version in versions: + build = build_class(version, args) + build.install() + builds.append(build) if args.steps in {'modules', 'tests'}: for build in builds: