diff --git a/.github/scripts/install-tensorrt-rtx.sh b/.github/scripts/install-tensorrt-rtx.sh index ef5c2945f3..61a0ce2ae6 100644 --- a/.github/scripts/install-tensorrt-rtx.sh +++ b/.github/scripts/install-tensorrt-rtx.sh @@ -17,8 +17,8 @@ install_tensorrt_rtx() { curl -L https://developer.nvidia.com/downloads/trt/rtx_sdk/secure/1.0/TensorRT-RTX-1.0.0.21.Windows.win10.cuda-12.9.zip -o TensorRT-RTX-1.0.0.21.Windows.win10.cuda-12.9.zip unzip TensorRT-RTX-1.0.0.21.Windows.win10.cuda-12.9.zip rtx_lib_dir=${PWD}/TensorRT-RTX-1.0.0.21/lib - export LD_LIBRARY_PATH=${rtx_lib_dir}:$LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + export PATH=${rtx_lib_dir}:$PATH + echo "PATH: $PATH" if [[ ${install_wheel_or_not} == true ]]; then pip install TensorRT-RTX-1.0.0.21/python/tensorrt_rtx-1.0.0.21-${CPYTHON_TAG}-none-win_amd64.whl fi diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml index 4b18ef559d..f2b2c8acd7 100644 --- a/.github/workflows/build-test-linux-x86_64.yml +++ b/.github/workflows/build-test-linux-x86_64.yml @@ -79,7 +79,7 @@ jobs: use-rtx: false tests-py-torchscript-fe: - name: Test torchscript frontend [Python] + name: Test resnet50 and bert [Python] needs: [filter-matrix, build] strategy: fail-fast: false @@ -100,23 +100,18 @@ jobs: build-matrix: ${{ needs.filter-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + python tools/perf/resnet50.py + python tools/perf/bert.py tests-py-dynamo-converters: - name: Test dynamo converters [Python] + name: Test Qwen/Qwen3-0.6B [Python] needs: [filter-matrix, build] strategy: fail-fast: false @@ -137,208 +132,205 @@ jobs: build-matrix: ${{ needs.filter-matrix.outputs.matrix }} pre-script: ${{ matrix.pre-script }} script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/py/dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin_with_attrs.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_flashinfer_rmsnorm.py - popd + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + python tools/llm/run_llm.py --model Qwen/Qwen3-0.6B --prompt "Gemma is Google’s family of open-weights LLMs, launched in February 2024 (Gemma 1.0, 2B/7B sizes). Designed to be lightweight, efficient, can you please tell me the difference between gemma3 and deepseek? which one is better in performance?" --precision FP16 --num_tokens 128 --benchmark - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd + # tests-py-dynamo-fe: + # name: Test dynamo frontend [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-fe + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ + # popd - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-serde - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py - popd + # tests-py-dynamo-serde: + # name: Test dynamo export serde [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-serde + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py + # popd - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torch-compile-be - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd + # tests-py-torch-compile-be: + # name: Test torch compile backend [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-torch-compile-be + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py + # popd - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd + # tests-py-dynamo-core: + # name: Test dynamo core [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-core + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ + # popd - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - nvidia-smi - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true - popd + # tests-py-dynamo-cudagraphs: + # name: Test dynamo cudagraphs [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-cudagraphs + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # nvidia-smi + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true + # popd - tests-py-core: - name: Test core [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd + # tests-py-core: + # name: Test core [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-core + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py/core + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . + # popd concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} diff --git a/.github/workflows/build-test-linux-x86_64_rtx.yml b/.github/workflows/build-test-linux-x86_64_rtx.yml index 24d0169e2c..f4900cb2bd 100644 --- a/.github/workflows/build-test-linux-x86_64_rtx.yml +++ b/.github/workflows/build-test-linux-x86_64_rtx.yml @@ -78,7 +78,7 @@ jobs: use-rtx: true tests-py-torchscript-fe: - name: Test torchscript frontend [Python] + name: Test resnet50 and bert [Python] needs: [filter-matrix, build] strategy: fail-fast: false @@ -100,23 +100,18 @@ jobs: pre-script: ${{ matrix.pre-script }} use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + python tools/perf/resnet50.py + python tools/perf/bert.py tests-py-dynamo-converters: - name: Test dynamo converters [Python] + name: Test Qwen/Qwen3-0.6B [Python] needs: [filter-matrix, build] strategy: fail-fast: false @@ -138,214 +133,211 @@ jobs: pre-script: ${{ matrix.pre-script }} use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/py/dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin_with_attrs.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_flashinfer_rmsnorm.py - popd + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + python tools/llm/run_llm.py --model Qwen/Qwen3-0.6B --prompt "Gemma is Google’s family of open-weights LLMs, launched in February 2024 (Gemma 1.0, 2B/7B sizes). Designed to be lightweight, efficient, can you please tell me the difference between gemma3 and deepseek? which one is better in performance?" --precision FP16 --num_tokens 128 --benchmark - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-fe - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd + # tests-py-dynamo-fe: + # name: Test dynamo frontend [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-fe + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ + # popd - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-serde - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py - popd + # tests-py-dynamo-serde: + # name: Test dynamo export serde [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-serde + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py + # popd - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-torch-compile-be - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd + # tests-py-torch-compile-be: + # name: Test torch compile backend [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-torch-compile-be + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py + # popd - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd + # tests-py-dynamo-core: + # name: Test dynamo core [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-core + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ + # popd - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - nvidia-smi - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true - popd + # tests-py-dynamo-cudagraphs: + # name: Test dynamo cudagraphs [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-dynamo-cudagraphs + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # nvidia-smi + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py || true + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py || true + # popd - tests-py-core: - name: Test core [Python] - needs: [filter-matrix, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - pre-script: packaging/pre_build_script.sh - post-script: packaging/post_build_script.sh - smoke-test-script: packaging/smoke_test_script.sh - uses: ./.github/workflows/linux-test.yml - with: - job-name: tests-py-core - repository: "pytorch/tensorrt" - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - pre-script: ${{ matrix.pre-script }} - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd + # tests-py-core: + # name: Test core [Python] + # needs: [filter-matrix, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # pre-script: packaging/pre_build_script.sh + # post-script: packaging/post_build_script.sh + # smoke-test-script: packaging/smoke_test_script.sh + # uses: ./.github/workflows/linux-test.yml + # with: + # job-name: tests-py-core + # repository: "pytorch/tensorrt" + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + # pre-script: ${{ matrix.pre-script }} + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py/core + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . + # popd concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml index 2d402a8799..a88adbe682 100644 --- a/.github/workflows/build-test-windows.yml +++ b/.github/workflows/build-test-windows.yml @@ -85,8 +85,9 @@ jobs: trigger-event: ${{ github.event_name }} timeout: 120 + tests-py-torchscript-fe: - name: Test torchscript frontend [Python] + name: Test Restnet/Bert [Python] needs: [substitute-runner, build] strategy: fail-fast: false @@ -103,23 +104,21 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.substitute-runner.outputs.matrix }} pre-script: packaging/driver_upgrade.bat + use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd + export USE_TRT_RTX=false + echo $LD_LIBRARY_PATH + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + packaging/vc_env_helper.bat python tools/perf/resnet50.py + packaging/vc_env_helper.bat python tools/perf/bert.py tests-py-dynamo-converters: - name: Test dynamo converters [Python] + name: Test Qwen/Qwen3-0.6B [Python] needs: [substitute-runner, build] strategy: fail-fast: false @@ -136,188 +135,191 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.substitute-runner.outputs.matrix }} pre-script: packaging/driver_upgrade.bat + use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd + export USE_TRT_RTX=false + echo $LD_LIBRARY_PATH + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + packaging/vc_env_helper.bat python tools/llm/run_llm.py --model Qwen/Qwen3-0.6B --prompt "Gemma is Google’s family of open-weights LLMs, launched in February 2024 (Gemma 1.0, 2B/7B sizes). Designed to be lightweight, efficient, can you please tell me the difference between gemma3 and deepseek? which one is better in performance?" --precision FP16 --num_tokens 128 --benchmark - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-serde - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py - popd + # tests-py-dynamo-fe: + # name: Test dynamo frontend [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-fe + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ + # popd - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torch-compile-be - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd + # tests-py-dynamo-serde: + # name: Test dynamo export serde [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-serde + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py + # popd - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd + # tests-py-torch-compile-be: + # name: Test torch compile backend [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-torch-compile-be + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py + # ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py + # popd - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py - popd + # tests-py-dynamo-core: + # name: Test dynamo core [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-core + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ + # ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ + # popd - tests-py-core: - name: Test core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd + # tests-py-dynamo-cudagraphs: + # name: Test dynamo cudagraphs [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-cudagraphs + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py + # popd + + # tests-py-core: + # name: Test core [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-core + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py/core + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . + # popd concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} diff --git a/.github/workflows/build-test-windows_rtx.yml b/.github/workflows/build-test-windows_rtx.yml index 6321cd8a52..044320b6a8 100644 --- a/.github/workflows/build-test-windows_rtx.yml +++ b/.github/workflows/build-test-windows_rtx.yml @@ -88,7 +88,7 @@ jobs: timeout: 120 tests-py-torchscript-fe: - name: Test torchscript frontend [Python] + name: Test Restnet/Bert [Python] needs: [substitute-runner, build] strategy: fail-fast: false @@ -107,22 +107,19 @@ jobs: pre-script: packaging/driver_upgrade.bat use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/modules - python hub.py - popd - pushd . - cd tests/py/ts - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/ - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/ - popd + export USE_TRT_RTX=true + echo $LD_LIBRARY_PATH + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + packaging/vc_env_helper.bat python tools/perf/resnet50.py + packaging/vc_env_helper.bat python tools/perf/bert.py tests-py-dynamo-converters: - name: Test dynamo converters [Python] + name: Test Qwen/Qwen3-0.6B [Python] needs: [substitute-runner, build] strategy: fail-fast: false @@ -141,193 +138,194 @@ jobs: pre-script: packaging/driver_upgrade.bat use-rtx: true script: | - set -euo pipefail + set -x export USE_HOST_DEPS=1 export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/ - popd + export USE_TRT_RTX=true + echo $LD_LIBRARY_PATH + nvidia-smi + python -c "import torch; print(torch.cuda.get_device_capability())" + python -c "import torch; print(torch.cuda.get_device_properties(0))" + packaging/vc_env_helper.bat python tools/llm/run_llm.py --model Qwen/Qwen3-0.6B --prompt "Gemma is Google’s family of open-weights LLMs, launched in February 2024 (Gemma 1.0, 2B/7B sizes). Designed to be lightweight, efficient, can you please tell me the difference between gemma3 and deepseek? which one is better in performance?" --precision FP16 --num_tokens 128 --benchmark - tests-py-dynamo-fe: - name: Test dynamo frontend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-fe - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ - popd + # tests-py-dynamo-fe: + # name: Test dynamo frontend [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-fe + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/ + # popd - tests-py-dynamo-serde: - name: Test dynamo export serde [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-serde - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py - popd + # tests-py-dynamo-serde: + # name: Test dynamo export serde [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-serde + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/reexport_test_results.xml --ir dynamo models/test_reexport.py + # popd - tests-py-torch-compile-be: - name: Test torch compile backend [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-torch-compile-be - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py - ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py - popd + # tests-py-torch-compile-be: + # name: Test torch compile backend [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-torch-compile-be + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_complete_be_e2e_test_results.xml --ir torch_compile models/test_models.py + # ../../../packaging/vc_env_helper.bat python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py + # popd - tests-py-dynamo-core: - name: Test dynamo core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ - ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ - popd + # tests-py-dynamo-core: + # name: Test dynamo core [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-core + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml --ignore runtime/test_002_cudagraphs_py.py --ignore runtime/test_002_cudagraphs_cpp.py runtime/ + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/ + # ../../../packaging/vc_env_helper.bat python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/ + # popd - tests-py-dynamo-cudagraphs: - name: Test dynamo cudagraphs [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-dynamo-cudagraphs - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py - cd dynamo - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py - python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py - popd + # tests-py-dynamo-cudagraphs: + # name: Test dynamo cudagraphs [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-dynamo-cudagraphs + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py + # cd dynamo + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_cpp_test_results.xml runtime/test_002_cudagraphs_cpp.py + # python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_cudagraphs_py_test_results.xml runtime/test_002_cudagraphs_py.py + # popd - tests-py-core: - name: Test core [Python] - needs: [substitute-runner, build] - strategy: - fail-fast: false - matrix: - include: - - repository: pytorch/tensorrt - package-name: torch_tensorrt - uses: ./.github/workflows/windows-test.yml - with: - job-name: tests-py-core - repository: ${{ matrix.repository }} - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.substitute-runner.outputs.matrix }} - pre-script: packaging/driver_upgrade.bat - use-rtx: true - script: | - set -euo pipefail - export USE_HOST_DEPS=1 - export CI_BUILD=1 - pushd . - cd tests/py/core - python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . - popd + # tests-py-core: + # name: Test core [Python] + # needs: [substitute-runner, build] + # strategy: + # fail-fast: false + # matrix: + # include: + # - repository: pytorch/tensorrt + # package-name: torch_tensorrt + # uses: ./.github/workflows/windows-test.yml + # with: + # job-name: tests-py-core + # repository: ${{ matrix.repository }} + # ref: "" + # test-infra-repository: pytorch/test-infra + # test-infra-ref: main + # build-matrix: ${{ needs.substitute-runner.outputs.matrix }} + # pre-script: packaging/driver_upgrade.bat + # use-rtx: true + # script: | + # set -euo pipefail + # export USE_HOST_DEPS=1 + # export CI_BUILD=1 + # pushd . + # cd tests/py/core + # python -m pytest -ra -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml . + # popd concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-tensorrt-rtx-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}-${{ inputs.job-name }} diff --git a/tools/perf/bert.py b/tools/perf/bert.py new file mode 100644 index 0000000000..9f8817026f --- /dev/null +++ b/tools/perf/bert.py @@ -0,0 +1,95 @@ +import tokenize +from pickle import FALSE, TRUE +from time import perf_counter + +import torch +import torch_tensorrt +from torch.export import Dim +from transformers import BertModel, BertTokenizer + +dtypes = [torch.float32, torch.float16, torch.bfloat16] +dynamic_shape_supports = [False] +iterations = 20 + +results = {} + + +def run_bert(dtype: torch.dtype): + model = BertModel.from_pretrained("bert-base-uncased").cuda().eval().to(dtype) + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + inputs = tokenizer("Hello, how are you?", return_tensors="pt").to("cuda") + input_ids = inputs["input_ids"] + token_type_ids = inputs["token_type_ids"] + inputs = (input_ids, token_type_ids) + exp_program = torch.export.export(model, args=inputs) + compile_start = perf_counter() + compiled_model = torch_tensorrt.dynamo.compile( + exp_program, + inputs=inputs, + use_explicit_type=True, + min_block_size=1, + immutable_weights=True, + cache_built_engines=False, + reuse_cached_engines=False, + use_python_runtime=False, + ) + compile_end = perf_counter() + compile_time = compile_end - compile_start + + with torch.no_grad(): + start = perf_counter() + compiled_model(*inputs) + end = perf_counter() + first_inference_time = end - start + start = perf_counter() + for i in range(iterations): + compiled_model(*inputs) + end = perf_counter() + second_inference_time = (end - start) / iterations + return compile_time, first_inference_time, second_inference_time + + +for dtype in dtypes: + results[dtype] = {} + for dynamic_shape_support in dynamic_shape_supports: + results[dtype][dynamic_shape_support] = {} + total_compile_time = 0 + total_first_inference_time = 0 + total_second_inference_time = 0 + for i in range(iterations): + compile_time, first_inference_time, second_inference_time = run_bert(dtype) + total_compile_time += compile_time + total_first_inference_time += first_inference_time + total_second_inference_time += second_inference_time + + results[dtype][dynamic_shape_support]["compile_time"] = ( + total_compile_time / iterations + ) + results[dtype][dynamic_shape_support]["first_inference_time"] = ( + total_first_inference_time / iterations + ) + results[dtype][dynamic_shape_support]["second_inference_time"] = ( + total_second_inference_time / iterations + ) + +for dype in dtypes: + for dynamic_shape_support in dynamic_shape_supports: + print(f"") + print( + f"================================================================================" + ) + print(f"========Bert model: {dtype=} {dynamic_shape_support=}=========") + print(f"compile_time: {results[dtype][dynamic_shape_support]['compile_time']}") + print( + f"first_inference_time: {results[dtype][dynamic_shape_support]['first_inference_time']}" + ) + print( + f"second_inference_time: {results[dtype][dynamic_shape_support]['second_inference_time']}" + ) + print( + f"================================================================================" + ) + print( + f"================================================================================" + ) + print(f"") diff --git a/tools/perf/resnet50.py b/tools/perf/resnet50.py new file mode 100644 index 0000000000..5e290e42c0 --- /dev/null +++ b/tools/perf/resnet50.py @@ -0,0 +1,99 @@ +from pickle import FALSE, TRUE +from time import perf_counter + +import torch +import torch_tensorrt +import torchvision.models as models +from torch.export import Dim + +dtypes = [torch.float32, torch.float16, torch.bfloat16] +dynamic_shape_supports = [True, False] + +iterations = 20 + +results = {} + + +def run_resnet(dtype: torch.dtype, dynamic_shape_support: bool): + model = models.resnet50(pretrained=True).to(dtype).eval().to("cuda") + inputs = (torch.randn((100, 3, 224, 224)).to(dtype).to("cuda"),) + if dynamic_shape_support: + batch_dim = Dim("batch_dim", min=2, max=100) + exp_program = torch.export.export( + model, args=inputs, dynamic_shapes=({0: batch_dim},) + ) + else: + exp_program = torch.export.export(model, args=inputs) + compile_start = perf_counter() + compiled_model = torch_tensorrt.dynamo.compile( + exp_program, + inputs=inputs, + use_explicit_type=True, + min_block_size=1, + immutable_weights=True, + cache_built_engines=False, + reuse_cached_engines=False, + use_python_runtime=False, + ) + compile_end = perf_counter() + compile_time = compile_end - compile_start + + with torch.no_grad(): + start = perf_counter() + compiled_model(*inputs) + end = perf_counter() + first_inference_time = end - start + start = perf_counter() + for i in range(iterations): + compiled_model(*inputs) + end = perf_counter() + second_inference_time = (end - start) / iterations + return compile_time, first_inference_time, second_inference_time + + +for dtype in dtypes: + results[dtype] = {} + for dynamic_shape_support in dynamic_shape_supports: + results[dtype][dynamic_shape_support] = {} + total_compile_time = 0 + total_first_inference_time = 0 + total_second_inference_time = 0 + for i in range(iterations): + compile_time, first_inference_time, second_inference_time = run_resnet( + dtype, dynamic_shape_support + ) + total_compile_time += compile_time + total_first_inference_time += first_inference_time + total_second_inference_time += second_inference_time + + results[dtype][dynamic_shape_support]["compile_time"] = ( + total_compile_time / iterations + ) + results[dtype][dynamic_shape_support]["first_inference_time"] = ( + total_first_inference_time / iterations + ) + results[dtype][dynamic_shape_support]["second_inference_time"] = ( + total_second_inference_time / iterations + ) + +for dype in dtypes: + for dynamic_shape_support in dynamic_shape_supports: + print(f"") + print( + f"================================================================================" + ) + print(f"========Resnet50 model: {dtype=} {dynamic_shape_support=}=========") + print(f"compile_time: {results[dtype][dynamic_shape_support]['compile_time']}") + print( + f"first_inference_time: {results[dtype][dynamic_shape_support]['first_inference_time']}" + ) + print( + f"second_inference_time: {results[dtype][dynamic_shape_support]['second_inference_time']}" + ) + print( + f"================================================================================" + ) + print( + f"================================================================================" + ) + print(f"")