Skip to content
build_test

feat: add URL scanning support to PlanResolver trait #1196

Sign in to view logs

feat: add URL scanning support to PlanResolver trait

feat: add URL scanning support to PlanResolver trait #1196

Workflow file for this run

.github/workflows/build_test.yml at fd0140a
name: build_test
on:
pull_request:
types: [opened, synchronize]
jobs:
check-rust:
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
components: rustfmt, clippy
- uses: prefix-dev/setup-pixi@v0.8.9
with:
cache: true
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-check-rust
path: |
~/.cargo
target
.pixi
- name: Check fmt
run: pixi run check-rs-fmt
- name: Check warnings
run: pixi run check-rs-warnings
- name: Check clippy
run: pixi run check-rs-clippy
test-rust:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-22.04
- windows-2022
- macos-14
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-test-rust2
path: |
~/.cargo
target
.pixi
- name: start minio server in the background
run: pixi run start-minio &
- name: Test rust
run: |
pixi run test-rs --release
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: failure()
with:
name: vegafusion-rt-test-images-${{ matrix.os }}
path: |
vegafusion-runtime/tests/output
build-vegafusion-wasm:
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
path: |
~/.cargo
target
.pixi
- name: Build package
run: |
pixi run pack-wasm
mkdir -p package
cp vegafusion-wasm/pkg/vegafusion-wasm-*.tgz package/
echo "timestamp: $(date)" > package/info.txt
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-wasm-package
path: package
check-vegafusion-python:
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
path: |
~/.cargo
target
.pixi
- name: Dev python
run: |
pixi run dev-py
- name: Check format and lint
run: |
pixi run lint-check-py
- name: Type check
run: |
pixi run type-check-py
- name: Verify generated proto files are up to date
run: |
pixi run gen-proto-py
git diff --exit-code vegafusion-python/vegafusion/proto/
# Use maturin action to build linux wheels within proper manylinux compatible containers
# (This is why we don't use the pixi "build-py" action)
build-vegafusion-python-linux-64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Clear target/wheels
run: rm -rf target/wheels
- name: Build wheels (Linux)
uses: messense/maturin-action@v1.45.0
with:
command: build
manylinux: 2014
rust-toolchain: stable
args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip --sdist -i python3.9
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: python-wheels-linux-64
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
retention-days: 1
if-no-files-found: error
build-vegafusion-python-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Clear target/wheels
run: rm -rf target/wheels
- name: Build arm64 wheels
uses: messense/maturin-action@v1.45.0
with:
command: build
manylinux: 2_28
rust-toolchain: stable
args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip --target aarch64-unknown-linux-gnu -i python3.9
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: python-wheels-linux-arm64
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
retention-days: 1
if-no-files-found: error
build-vegafusion-python-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-win-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: python-wheels-win-64
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
retention-days: 1
if-no-files-found: error
build-vegafusion-python-osx-64:
runs-on: macos-15-intel
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: python-wheels-osx-64
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
retention-days: 1
if-no-files-found: error
build-vegafusion-python-osx-arm64:
runs-on: macos-14
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
targets: aarch64-apple-darwin
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py --target aarch64-apple-darwin
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: python-wheels-osx-arm64
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
retention-days: 1
if-no-files-found: error
test-vegafusion-python-linux-64:
runs-on: ubuntu-22.04
needs: [build-vegafusion-python-linux-64]
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Install Chrome
uses: browser-actions/setup-chrome@v1
with:
chrome-version: stable
- name: Install fonts on Linux
run: |
echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
sudo apt-get install ttf-mscorefonts-installer
- name: Download vegafusion-python wheel
uses: actions/download-artifact@v4.1.8
with:
name: python-wheels-linux-64
path: target/wheels/
- name: Install wheel and chromedriver
run: |
pixi run pip install target/wheels/vegafusion-*manylinux_2_17_x86_64*.whl
pixi run pip install chromedriver-binary-auto
- name: Test lazy imports
working-directory: vegafusion-python/
run: pixi run python checks/check_lazy_imports.py
- name: Test vegafusion
working-directory: vegafusion-python/
env:
VEGAFUSION_TEST_HEADLESS: 1
run: pixi run pytest
- name: Upload test artifacts
uses: actions/upload-artifact@v4.4.3
if: failure()
with:
name: vegafusion-python-test-failures
path: |
vegafusion-python/tests/output/*
test-vegafusion-python-osx-arm64:
runs-on: macos-14
needs: [build-vegafusion-python-osx-arm64]
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Setup Python
uses: actions/setup-python@v5.3.0
with:
python-version: "3.10"
- name: Download vegafusion-python wheel
uses: actions/download-artifact@v4.1.8
with:
name: python-wheels-osx-arm64
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
run: |
ls -la
python -m pip install vegafusion-*macosx_11_*_arm64.whl
# Optional dependencies
python -m pip install pyarrow pandas polars "duckdb>=1.0" vl-convert-python "pandas>=2.2" "protobuf==5.29.3"
# Test dependencies
python -m pip install pytest altair vega-datasets scikit-image inline-snapshot
- name: Test vegafusion
working-directory: vegafusion-python/
run: pytest --ignore=tests/test_jupyter_widget.py
test-vegafusion-python-win-64:
runs-on: windows-2022
needs: [build-vegafusion-python-win-64]
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Setup Python
uses: actions/setup-python@v5.3.0
with:
python-version: "3.9"
- name: Download vegafusion-python wheel
uses: actions/download-artifact@v4.1.8
with:
name: python-wheels-win-64
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
shell: powershell
run: |
Get-ChildItem -Force
$vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" | Select-Object -First 1
python -m pip install $vegafusion
# Optional dependencies
python -m pip install pyarrow pandas "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python "protobuf==5.29.3"
# Test dependencies
python -m pip install pytest altair vega-datasets scikit-image inline-snapshot
- name: Test vegafusion
working-directory: vegafusion-python/
run: pytest --ignore=tests/test_jupyter_widget.py
build-vegafusion-server-linux-64:
runs-on: ubuntu-22.04
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-linux-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-server
- name: zip executable
uses: vimtor/action-zip@v1.2
with:
files: target/release-opt/vegafusion-server
dest: vegafusion-server-linux-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-server-linux-64
path: |
vegafusion-server-*
retention-days: 1
if-no-files-found: error
# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
# # cargo (without involving conda).
#
# build-vegafusion-server-linux-arm64:
# runs-on: ubuntu-22.04
# steps:
# - name: Check out repository code
# uses: actions/checkout@v4.2.2
# - name: Install pixi
# run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
# - name: Cache
# uses: actions/cache@v4
# with:
# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-server
# path: |
# ~/.cargo
# target
# .pixi
# - name: Build vegafusion-server
# run: |
# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
# pixi add gcc_linux-aarch64 -p linux-64
# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
# pixi run build-rs-server --features=protobuf-src --target aarch64-unknown-linux-gnu
# - name: zip executable
# uses: vimtor/action-zip@v1.2
# with:
# files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
# dest: vegafusion-server-linux-arm64.zip
# - name: Upload artifacts
# uses: actions/upload-artifact@v4.4.3
# with:
# name: vegafusion-server-linux-arm64
# path: |
# vegafusion-server-*
build-vegafusion-server-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install latest stable Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- name: Cache rust dependencies
uses: Swatinem/rust-cache@v2.7.5
with:
cache-on-failure: True
- name: Download arm64 toolchain
run: |
rustup target add aarch64-unknown-linux-gnu
- name: Download gcc for cross compilation
run: |
sudo apt-get update
sudo apt-get install gcc-aarch64-linux-gnu
- name: Build vegafusion-server
env:
RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
run: |
cargo build -p vegafusion-server --features=protobuf-src --profile release-opt --target=aarch64-unknown-linux-gnu
- name: zip executable
uses: vimtor/action-zip@v1.2
with:
files: target/aarch64-unknown-linux-gnu/release-opt/vegafusion-server
dest: vegafusion-server-linux-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-server-linux-arm64
path: |
vegafusion-server-*
retention-days: 1
if-no-files-found: error
build-vegafusion-server-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-win-64b
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi install -vvv
pixi run build-rs-server
- name: zip executable
uses: vimtor/action-zip@v1.2
with:
files: target/release-opt/vegafusion-server.exe
dest: vegafusion-server-win-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-server-win-64
path: |
vegafusion-server-*
retention-days: 1
if-no-files-found: error
build-vegafusion-server-osx-64:
runs-on: macos-15-intel
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-server
- name: zip executable
uses: vimtor/action-zip@v1.2
with:
files: target/release-opt/vegafusion-server
dest: vegafusion-server-osx-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-server-osx-64
path: |
vegafusion-server-*
retention-days: 1
if-no-files-found: error
build-vegafusion-server-osx-arm64:
runs-on: macos-14
steps:
- name: Check out repository code
uses: actions/checkout@v4.2.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: stable
targets: aarch64-apple-darwin
- uses: prefix-dev/setup-pixi@v0.8.9
- name: Cache
uses: actions/cache@v4
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-server --target aarch64-apple-darwin
- name: zip executable
uses: vimtor/action-zip@v1.2
with:
files: target/aarch64-apple-darwin/release-opt/vegafusion-server
dest: vegafusion-server-osx-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@v4.4.3
with:
name: vegafusion-server-osx-arm64
path: |
vegafusion-server-*
retention-days: 1
if-no-files-found: error
# Bundle all server builds into a single artifact for easier downloading
bundle-vegafusion-server-builds:
needs:
[
build-vegafusion-server-linux-64,
build-vegafusion-server-linux-arm64,
build-vegafusion-server-win-64,
build-vegafusion-server-osx-64,
build-vegafusion-server-osx-arm64,
]
runs-on: ubuntu-latest
steps:
- name: Merge Server Artifacts
uses: actions/upload-artifact/merge@v4.4.3
with:
name: vegafusion-server-all
pattern: vegafusion-server-*
delete-merged: true
separate-directories: false
# Bundle all python wheels into a single artifact for easier downloading
bundle-vegafusion-python-wheels:
needs: [
# jobs that build but don't have test dependencies
build-vegafusion-python-linux-arm64,
build-vegafusion-python-osx-64,
# jobs that test the wheels
test-vegafusion-python-linux-64,
test-vegafusion-python-osx-arm64,
test-vegafusion-python-win-64,
]
runs-on: ubuntu-latest
steps:
- name: Merge Wheel Artifacts
uses: actions/upload-artifact/merge@v4.4.3
with:
name: vegafusion-python-wheels-all
pattern: python-wheels-*
delete-merged: true
separate-directories: false