diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 0000000..f87dff4 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,173 @@ +name: E2E Tests + +concurrency: + group: e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + branches: + - '**' + types: [ opened, synchronize, reopened, ready_for_review ] + + workflow_dispatch: + inputs: + verbose: + description: "Output more information when triggered manually" + required: false + default: "" + +env: + CARGO_TERM_COLOR: always + VERBOSE: ${{ github.event.inputs.verbose }} + +# job to run tests in parallel +jobs: + # Looking for e2e tests + find-tests: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft == false }} + outputs: + test-files: ${{ steps.get-tests.outputs.test-files }} + steps: + - name: Check-out repository under $GITHUB_WORKSPACE + uses: actions/checkout@v4 + + - name: Find test files + id: get-tests + run: | + test_files=$(find tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))') + # keep it here for future debug + # test_files=$(find tests/e2e_tests -type f -name "test*.py" | grep -E 'test_(hotkeys|staking)\.py$' | jq -R -s -c 'split("\n") | map(select(. != ""))') + echo "Found test files: $test_files" + echo "test-files=$test_files" >> "$GITHUB_OUTPUT" + shell: bash + + # Pull docker image + pull-docker-image: + runs-on: ubuntu-latest + outputs: + image-name: ${{ steps.set-image.outputs.image }} + steps: + - name: Set Docker image tag based on label or branch + id: set-image + run: | + echo "Event: $GITHUB_EVENT_NAME" + echo "Branch: $GITHUB_REF_NAME" + + echo "Reading labels ..." + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + labels=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH") + else + labels="" + fi + + image="" + + for label in $labels; do + echo "Found label: $label" + case "$label" in + "subtensor-localnet:main") + image="ghcr.io/opentensor/subtensor-localnet:main" + break + ;; + "subtensor-localnet:testnet") + image="ghcr.io/opentensor/subtensor-localnet:testnet" + break + ;; + "subtensor-localnet:devnet") + image="ghcr.io/opentensor/subtensor-localnet:devnet" + break + ;; + esac + done + + if [[ -z "$image" ]]; then + # fallback to default based on branch + if [[ "${GITHUB_REF_NAME}" == "master" ]]; then + image="ghcr.io/opentensor/subtensor-localnet:main" + else + image="ghcr.io/opentensor/subtensor-localnet:devnet-ready" + fi + fi + + echo "✅ Final selected image: $image" + echo "image=$image" >> "$GITHUB_OUTPUT" + + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Pull Docker Image + run: docker pull ${{ steps.set-image.outputs.image }} + + - name: Save Docker Image to Cache + run: docker save -o subtensor-localnet.tar ${{ steps.set-image.outputs.image }} + + - name: Upload Docker Image as Artifact + uses: actions/upload-artifact@v4 + with: + name: subtensor-localnet + path: subtensor-localnet.tar + + # Job to run tests in parallel + run-fast-blocks-e2e-test: + name: "FB: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}" + needs: + - find-tests + - pull-docker-image + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false # Allow other matrix jobs to run even if this job fails + max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner) + matrix: + os: + - ubuntu-latest + test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }} + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: install dependencies + run: | + uv sync --extra dev --dev + + - name: Download Cached Docker Image + uses: actions/download-artifact@v4 + with: + name: subtensor-localnet + + - name: Load Docker Image + run: docker load -i subtensor-localnet.tar + + - name: Run tests with retry + env: + LOCALNET_IMAGE_NAME: ${{ needs.pull-docker-image.outputs.image-name }} + run: | + for i in 1 2 3; do + echo "::group::🔁 Test attempt $i" + if uv run pytest ${{ matrix.test-file }} -s; then + echo "✅ Tests passed on attempt $i" + echo "::endgroup::" + exit 0 + else + echo "❌ Tests failed on attempt $i" + echo "::endgroup::" + if [ "$i" -lt 3 ]; then + echo "Retrying..." + sleep 5 + fi + fi + done + + echo "Tests failed after 3 attempts" + exit 1 diff --git a/.github/workflows/run-async-substrate-interface-tests.yml b/.github/workflows/run-async-substrate-interface-tests.yml deleted file mode 100644 index 9890192..0000000 --- a/.github/workflows/run-async-substrate-interface-tests.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Run Tests - -on: - push: - branches: [main, staging] - pull_request: - branches: [main, staging] - workflow_dispatch: - -jobs: - find-tests: - runs-on: ubuntu-latest - steps: - - name: Check-out repository - uses: actions/checkout@v4 - - - name: Find test files - id: get-tests - run: | - test_files=$(find tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))') - echo "::set-output name=test-files::$test_files" - - pull-docker-image: - runs-on: ubuntu-latest - steps: - - name: Log in to GitHub Container Registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - - - name: Pull Docker Image - run: docker pull ghcr.io/opentensor/subtensor-localnet:devnet-ready - - - name: Save Docker Image to Cache - run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:devnet-ready - - - name: Upload Docker Image as Artifact - uses: actions/upload-artifact@v4 - with: - name: subtensor-localnet - path: subtensor-localnet.tar - - run-unit-tests: - name: ${{ matrix.test-file }} / Python ${{ matrix.python-version }} - needs: - - find-tests - - pull-docker-image - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - max-parallel: 32 - matrix: - test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }} - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - - steps: - - name: Check-out repository - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - uv venv .venv - source .venv/bin/activate - uv pip install .[dev] - - - name: Download Docker Image - uses: actions/download-artifact@v4 - with: - name: subtensor-localnet - - - name: Load Docker Image - run: docker load -i subtensor-localnet.tar - - - name: Run pytest - run: | - source .venv/bin/activate - uv run pytest ${{ matrix.test-file }} -v -s \ No newline at end of file diff --git a/.github/workflows/unit-and-integration-test.yml b/.github/workflows/unit-and-integration-test.yml new file mode 100644 index 0000000..7bc70ae --- /dev/null +++ b/.github/workflows/unit-and-integration-test.yml @@ -0,0 +1,59 @@ +name: Unit and integration tests checker +permissions: + contents: read + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + +jobs: + unit-and-integration-tests: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + + strategy: + fail-fast: false + max-parallel: 5 + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Cache venv + id: cache + uses: actions/cache@v4 + with: + path: venv + key: v2-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} + + - name: Install deps + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + python -m pip install uv + python -m uv sync --extra dev --active + + - name: Unit tests + timeout-minutes: 20 + env: + PYTHONUNBUFFERED: "1" + run: | + source venv/bin/activate + python -m uv run pytest -n 2 tests/unit_tests/ --reruns 3 + + - name: Integration tests + timeout-minutes: 20 + env: + PYTHONUNBUFFERED: "1" + run: | + source venv/bin/activate + python -m uv run pytest -n 2 tests/integration_tests/ --reruns 3 diff --git a/pyproject.toml b/pyproject.toml index 0e9acb0..148b33a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,4 +47,13 @@ requires = ["setuptools~=70.0.0", "wheel"] build-backend = "setuptools.build_meta" [project.optional-dependencies] -dev = ["pytest", "bittensor", "pytest-asyncio"] +dev = [ + "bittensor", + "pytest==8.3.5", + "pytest-asyncio==0.26.0", + "pytest-mock==3.14.0", + "pytest-split==0.10.0", + "pytest-xdist==3.6.1", + "pytest-rerunfailures==10.2", + "substrate-interface" +] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_old_new.py b/tests/e2e_tests/test_old_new.py similarity index 100% rename from tests/test_old_new.py rename to tests/e2e_tests/test_old_new.py diff --git a/tests/test_substrate_addons.py b/tests/e2e_tests/test_substrate_addons.py similarity index 79% rename from tests/test_substrate_addons.py rename to tests/e2e_tests/test_substrate_addons.py index da671eb..c776506 100644 --- a/tests/test_substrate_addons.py +++ b/tests/e2e_tests/test_substrate_addons.py @@ -1,23 +1,24 @@ -import threading import subprocess +import time import pytest -import time from async_substrate_interface import AsyncSubstrateInterface, SubstrateInterface +from async_substrate_interface.errors import MaxRetriesExceeded, StateDiscardedError from async_substrate_interface.substrate_addons import ( RetrySyncSubstrate, RetryAsyncSubstrate, ) -from async_substrate_interface.errors import MaxRetriesExceeded, StateDiscardedError from tests.conftest import start_docker_container - -LATENT_LITE_ENTRYPOINT = "wss://lite.sub.latent.to:443" +from tests.helpers.settings import ARCHIVE_ENTRYPOINT, LATENT_LITE_ENTRYPOINT @pytest.fixture(scope="function") def docker_containers(): - processes = (start_docker_container(9945, 9945), start_docker_container(9946, 9946)) + processes = ( + start_docker_container(9944, "9944"), + start_docker_container(9945, "9945"), + ) try: yield processes @@ -29,7 +30,7 @@ def docker_containers(): @pytest.fixture(scope="function") def single_local_chain(): - process = start_docker_container(9945, 9945) + process = start_docker_container(9945, "9944") try: yield process finally: @@ -51,6 +52,11 @@ def test_retry_sync_substrate(single_local_chain): time.sleep(2) +@pytest.mark.skip( + "There's an issue with this running in the GitHub runner, " + "where it seemingly cannot connect to the docker container. " + "It does run locally, however." +) def test_retry_sync_substrate_max_retries(docker_containers): time.sleep(10) with RetrySyncSubstrate( @@ -72,30 +78,30 @@ def test_retry_sync_substrate_max_retries(docker_containers): def test_retry_sync_substrate_offline(): with pytest.raises(ConnectionError): RetrySyncSubstrate( - "ws://127.0.0.1:9945", fallback_chains=["ws://127.0.0.1:9946"] + "ws://127.0.0.1:9944", fallback_chains=["ws://127.0.0.1:9945"] ) @pytest.mark.asyncio async def test_retry_async_subtensor_archive_node(): - async with AsyncSubstrateInterface("wss://lite.sub.latent.to:443") as substrate: + async with AsyncSubstrateInterface(LATENT_LITE_ENTRYPOINT) as substrate: current_block = await substrate.get_block_number() old_block = current_block - 1000 with pytest.raises(StateDiscardedError): await substrate.get_block(block_number=old_block) async with RetryAsyncSubstrate( - "wss://lite.sub.latent.to:443", archive_nodes=["ws://178.156.172.75:9944"] + LATENT_LITE_ENTRYPOINT, archive_nodes=[ARCHIVE_ENTRYPOINT] ) as substrate: assert isinstance((await substrate.get_block(block_number=old_block)), dict) def test_retry_sync_subtensor_archive_node(): - with SubstrateInterface("wss://lite.sub.latent.to:443") as substrate: + with SubstrateInterface(LATENT_LITE_ENTRYPOINT) as substrate: current_block = substrate.get_block_number() old_block = current_block - 1000 with pytest.raises(StateDiscardedError): substrate.get_block(block_number=old_block) with RetrySyncSubstrate( - "wss://lite.sub.latent.to:443", archive_nodes=["ws://178.156.172.75:9944"] + LATENT_LITE_ENTRYPOINT, archive_nodes=[ARCHIVE_ENTRYPOINT] ) as substrate: assert isinstance((substrate.get_block(block_number=old_block)), dict) diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..299f47b --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = ignore::DeprecationWarning:pkg_resources.*: +asyncio_default_fixture_loop_scope = "session" \ No newline at end of file