Add action testing framework with test-runner and discoverable test files

.gitattributes

@@ -0,0 +1,4 @@
+# Exclude test directories from git archives (used for releases)
+# Note: This does not affect GitHub Actions 'uses:' references,
+# which clone the entire repository
+**/tests/ export-ignore

.github/workflows/action-tests.yml

@@ -0,0 +1,91 @@
+name: Action Tests
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  should:
+    runs-on: ubuntu-24.04
+    outputs:
+      run: ${{ steps.filter.outputs.run }}
+    steps:
+    - uses: envoyproxy/toolshed/gh-actions/github/should-run@c04bdf5c854da00a30144f3597967e73ca376f38
+      id: filter
+      with:
+        config: |
+          paths:
+          - 'gh-actions/**'
+          - '.github/workflows/action-tests.yml'
+          - '.github/workflows/tests/**'
+
+  discover-tests:
+    runs-on: ubuntu-24.04
+    needs: should
+    if: fromJSON(needs.should.outputs.run)
+    outputs:
+      tests: ${{ steps.find-tests.outputs.tests }}
+    steps:
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+    - id: find-tests
+      shell: bash
+      run: |
+        # Find all test files in gh-actions/*/tests/ directories
+        tests=$(find gh-actions -path "*/tests/*.test.yml" -type f \
+          | jq -R -s -c 'split("\n") | map(select(length > 0))')
+        echo "tests=$tests" >> $GITHUB_OUTPUT
+        echo "Found tests: $tests"
+
+  run-tests:
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    if: ${{ needs.discover-tests.outputs.tests != '[]' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJSON(needs.discover-tests.outputs.tests) }}
+    steps:
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+
+    - name: Load test config
+      id: load-test
+      shell: bash
+      run: |
+        config=$(cat "${{ matrix.test }}")
+        # Use a delimiter to handle multiline content
+        echo "config<<EOFCONFIG" >> $GITHUB_OUTPUT
+        echo "$config" >> $GITHUB_OUTPUT
+        echo "EOFCONFIG" >> $GITHUB_OUTPUT
+
+    - name: Run test
+      uses: envoyproxy/toolshed/gh-actions/test-runner@9bb6f3a6aa6bf95ae2e1c8676dbf7056ff6e7440
+      with:
+        config: ${{ steps.load-test.outputs.config }}
+        name: ${{ matrix.test }}
+
+  status:
+    runs-on: ubuntu-24.04
+    if: >-
+      always()
+      && github.event_name == 'pull_request'
+    name: Action Tests
+    needs:
+    - should
+    - discover-tests
+    - run-tests
+    steps:
+    - run: |
+        if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more jobs failed or were cancelled"
+            exit 1
+        fi
+        echo "All required jobs passed or were skipped"

README.md

@@ -15,7 +15,7 @@ Multi-language tooling and libraries for Envoy proxy's CI and development workfl
 - **[py/](py/)** - Python packages, libraries, runners, checkers, and dependencies - see [py/README.md](py/README.md) for details
 - **[bazel/](bazel/)** - Bazel build configurations and rules
 - **[rust/](rust/)** - Rust tooling and libraries
-- **[gh-actions/](gh-actions/)** - GitHub Actions workflows
+- **[gh-actions/](gh-actions/)** - GitHub Actions workflows (see [gh-actions/test-runner/README.md](gh-actions/test-runner/README.md) for testing actions)
 - **[sh/](sh/)** - Shell scripts and utilities
 
 ### Development

gh-actions/bson/tests/basic.test.yml

@@ -0,0 +1,16 @@
+# Example test for the bson action
+using: envoyproxy/toolshed/gh-actions/bson@actions-v0.3.35
+with:
+  input: |
+    message: "Hello World"
+    count: 42
+  input-format: yaml
+  filter: |
+    "echo \(.message)"
+    | bash::code
+
+after:
+- shell: bash
+  run: |
+    # The bson action should have executed the bash code
+    echo "✓ Bson action executed successfully"

gh-actions/diskspace/tests/basic.test.yml

@@ -0,0 +1,22 @@
+# Example test for the diskspace action with mocking
+before:
+- shell: bash
+  run: |
+    # Create a test directory to remove
+    mkdir -p /tmp/test-cruft
+    echo "test data" > /tmp/test-cruft/testfile.txt
+
+using: envoyproxy/toolshed/gh-actions/diskspace@actions-v0.3.35
+with:
+  to_remove: /tmp/test-cruft
+  background: false
+
+after:
+- shell: bash
+  run: |
+    # Verify the directory was removed
+    if [[ -d /tmp/test-cruft ]]; then
+      echo "Error: /tmp/test-cruft still exists"
+      exit 1
+    fi
+    echo "✓ Diskspace action successfully removed test directory"

gh-actions/fetch/tests/basic.test.yml

@@ -0,0 +1,41 @@
+# Example test for the fetch action
+before:
+- shell: bash
+  run: |
+    # Create a temporary file to serve
+    mkdir -p /tmp/test-server
+    echo "test content" > /tmp/test-server/test.txt
+    # Start a simple HTTP server in the background
+    cd /tmp/test-server
+    python3 -m http.server 8765 &
+    echo $! > /tmp/server.pid
+    sleep 2
+
+using: envoyproxy/toolshed/gh-actions/fetch@actions-v0.3.35
+with:
+  url: http://localhost:8765/test.txt
+  filename: downloaded.txt
+
+after:
+- shell: bash
+  run: |
+    # Check that the file was downloaded
+    if [[ ! -f downloaded.txt ]]; then
+      echo "Error: downloaded.txt not found"
+      exit 1
+    fi
+    # Check content
+    content=$(cat downloaded.txt)
+    if [[ "$content" != "test content" ]]; then
+      echo "Error: content mismatch"
+      echo "Expected: test content"
+      echo "Got: $content"
+      exit 1
+    fi
+    # Cleanup
+    kill $(cat /tmp/server.pid) || true
+    rm -rf /tmp/test-server /tmp/server.pid downloaded.txt
+    echo "✓ Fetch action test passed"
+
+outputs:
+  path: downloaded.txt

gh-actions/jq/tests/basic.test.yml

@@ -0,0 +1,9 @@
+# Example test for the jq action
+using: envoyproxy/toolshed/gh-actions/jq@actions-v0.3.35
+with:
+  input: '{"foo": "bar", "baz": 123}'
+  filter: .foo
+  options: -r
+
+outputs:
+  value: bar

gh-actions/test-runner/README.md

@@ -0,0 +1,148 @@
+# Test Runner
+
+A GitHub Action for testing other GitHub Actions with fixtures and side effect validation.
+
+## Overview
+
+This action provides a framework for testing GitHub Actions by:
+- Running setup steps (mocking commands, creating fixtures)
+- Executing the action under test
+- Validating outputs against expected values
+- Running cleanup/verification steps
+
+## Usage
+
+### Basic Test
+
+Create a test file in `gh-actions/<action-name>/tests/` with the following structure:
+
+```yaml
+using: <action-to-test>
+with:
+  <action-inputs>
+
+outputs:
+  <expected-outputs>
+```
+
+### Example: Testing jq action
+
+```yaml
+# gh-actions/jq/tests/basic.test.yml
+using: envoyproxy/toolshed/gh-actions/jq@actions-v0.3.35
+with:
+  input: '{"foo": "bar", "baz": 123}'
+  filter: .foo
+  options: -r
+
+outputs:
+  value: bar
+```
+
+### Advanced Test with Setup and Validation
+
+```yaml
+# gh-actions/fetch/tests/with-mock.test.yml
+before:
+- shell: bash
+  run: |
+    # Create mock environment
+    mkdir -p /tmp/mock/bin
+    cat > /tmp/mock/bin/gh << 'EOF'
+    #!/bin/bash
+    echo "${@}" >> /tmp/gh-commands.log
+    echo "mock output"
+    EOF
+    chmod +x /tmp/mock/bin/gh
+    export PATH="/tmp/mock/bin:$PATH"
+
+using: envoyproxy/toolshed/gh-actions/fetch@actions-v0.3.35
+with:
+  url: http://example.com/file.txt
+  filename: test.txt
+
+after:
+- shell: bash
+  run: |
+    # Verify side effects
+    if [[ ! -f /tmp/gh-commands.log ]]; then
+      echo "Expected gh command was not called"
+      exit 1
+    fi
+    # Cleanup
+    rm -rf /tmp/mock /tmp/gh-commands.log
+
+outputs:
+  path: test.txt
+```
+
+## Test Configuration Schema
+
+### Fields
+
+- **before** (optional): Array of steps to run before the action under test
+  - Used for setting up mocks, creating fixtures, etc.
+  - Each step should be a valid GitHub Actions step (with `shell` and `run`, or `uses`)
+
+- **using** (required): The action to test (e.g., `envoyproxy/toolshed/gh-actions/jq@actions-v0.3.35`)
+
+- **with** (optional): Object containing inputs to pass to the action
+
+- **after** (optional): Array of steps to run after the action under test
+  - Used for verification, cleanup, etc.
+  - Steps can fail the test by exiting with non-zero status
+
+- **outputs** (optional): Object containing expected outputs
+  - Keys are output names, values are expected values
+  - If specified, actual outputs will be compared against these expected values
+
+### Environment Variable Interpolation
+
+Test configurations support environment variable interpolation for parameterized testing:
+
+```yaml
+# Can use environment variables in test configs
+using: envoyproxy/toolshed/gh-actions/jq@${ACTION_VERSION}
+with:
+  input: ${TEST_INPUT}
+  filter: .foo
+
+outputs:
+  value: ${EXPECTED_OUTPUT}
+```
+
+Set these variables in the workflow that runs the tests, or in a matrix strategy for multiple test variations.
+
+## How It Works
+
+1. **Parse Configuration**: The test-runner parses the YAML test configuration
+2. **Run Before Steps**: If `before` is defined, those steps are executed
+3. **Create Wrapper**: A temporary action is created that wraps the action under test
+4. **Run Action**: The action under test is executed with the specified inputs
+5. **Run After Steps**: If `after` is defined, those steps are executed
+6. **Validate Outputs**: If `outputs` are specified, they're compared against actual outputs
+7. **Cleanup**: Temporary files are removed
+
+## CI Integration
+
+Tests are automatically discovered and run by the `action-tests.yml` workflow:
+
+```yaml
+# .github/workflows/action-tests.yml
+- Find all *.test.yml files in gh-actions/*/tests/ directories
+- Run each test in a separate matrix job
+- Report results
+```
+
+## Best Practices
+
+1. **Keep tests focused**: Test one aspect of the action at a time
+2. **Use descriptive names**: Name test files clearly (e.g., `filter-arrays.test.yml`, `with-options.test.yml`)
+3. **Clean up**: Always clean up any resources created in `before` steps
+4. **Mock external dependencies**: Use the `before` section to mock commands like `gh`, `curl`, etc.
+5. **Verify side effects**: Use the `after` section to check files, logs, or other side effects
+6. **Multiple tests**: Create multiple test files in the `tests/` directory for different scenarios
+
+## Examples
+
+See `gh-actions/*/tests/` directories for example test cases (e.g., `gh-actions/jq/tests/`, `gh-actions/test-runner/tests/`).