[Liger-Kernels] Fix Liger-Kernels tests hanging (#5382)

Resolves
#5292

There's also refactoring of Liger-Kernels installation to be consistent
with sglang, vllm

PR depends on:
- [x] #5376

Before merging:
- [x] Merge
#5376
- [x] Test third-party testing CI, there should be no hanging for liger
kernels
- [x] Test third-party benchmarks CI

Author: Egor-Krivov

.github/workflows/third-party-benchmarks.yml

@@ -104,21 +104,18 @@ jobs:
           python transform_results.py $REPORTS/moe-gemm-performance.csv $REPORTS/moe-gemm-fp8-report.csv --tag $TAG --benchmark moe-fp8-benchmark
 
       - name: Run Liger-Kernel benchmarks
-        if: ${{ steps.install.outcome == 'success' && !cancelled() && (inputs.benchmarks == '' || contains(fromJson(inputs.benchmarks || '[]'), 'liger-kernel')) }}
+        if: ${{ steps.install.outcome == 'success' && !cancelled() && (inputs.benchmarks == '' || contains(fromJson(inputs.benchmarks || '[]'), 'liger')) }}
         run: |
           source ./scripts/capture-hw-details.sh
 
-          cd benchmarks/third_party/liger_kernels
-
-          git clone https://github.com/linkedin/Liger-Kernel
-          pip install -e Liger-Kernel
+          ./scripts/test-triton.sh --install-liger --skip-pip-install --skip-pytorch-install
 
           # To remember return code, but still copy results
           RET_CODE=0
-          bash ./run_benchmarks.sh || RET_CODE=$?
+          bash benchmarks/third_party/liger/run_benchmarks.sh || RET_CODE=$?
 
           cp Liger-Kernel/benchmark/data/all_benchmark_data.csv $REPORTS/liger-raw.csv
-          python transform.py $REPORTS/liger-raw.csv $REPORTS/liger-report.csv --tag $TAG
+          python benchmarks/third_party/liger/transform.py $REPORTS/liger-raw.csv $REPORTS/liger-report.csv --tag $TAG
 
           # Return the captured return code at the end
           exit "$RET_CODE"

benchmarks/third_party/liger_kernels/README.md renamed to benchmarks/third_party/liger/README.md

@@ -26,11 +26,12 @@ TEST:
     --flex-attention
     --instrumentation
     --inductor
-    --sglang
-    --liger
     --vllm
     --install-vllm
+    --sglang
     --install-sglang
+    --liger
+    --install-liger
 
 OPTION:
     --unskip
@@ -72,10 +73,11 @@ TEST_BENCHMARK_FLEX_ATTENTION=false
 TEST_INSTRUMENTATION=false
 TEST_INDUCTOR=false
 TEST_SGLANG=false
+INSTALL_SGLANG=false
 TEST_LIGER=false
+INSTALL_LIGER=false
 TEST_VLLM=false
 INSTALL_VLLM=false
-INSTALL_SGLANG=false
 TEST_TRITON_KERNELS=false
 VENV=false
 TRITON_TEST_REPORTS=false
@@ -192,13 +194,13 @@ while (( $# != 0 )); do
       TEST_DEFAULT=false
       shift
       ;;
-    --install-sglang)
-      INSTALL_SGLANG=true
+    --sglang)
+      TEST_SGLANG=true
       TEST_DEFAULT=false
       shift
       ;;
-    --sglang)
-      TEST_SGLANG=true
+    --install-sglang)
+      INSTALL_SGLANG=true
       TEST_DEFAULT=false
       shift
       ;;
@@ -207,6 +209,11 @@ while (( $# != 0 )); do
       TEST_DEFAULT=false
       shift
       ;;
+    --install-liger)
+      INSTALL_LIGER=true
+      TEST_DEFAULT=false
+      shift
+      ;;
     --vllm)
       TEST_VLLM=true
       TEST_DEFAULT=false
@@ -596,6 +603,10 @@ run_inductor_tests() {
   grep AlbertForMaskedLM inductor_log.csv | grep -q ,pass,
 }
 
+run_test_deps_install() {
+  pip install pytest pytest-cov pytest-xdist
+}
+
 run_sglang_install() {
   echo "************************************************"
   echo "******    Installing SGLang                 ****"
@@ -620,8 +631,6 @@ run_sglang_install() {
     pip install -e "./python"
     cd ..
   fi
-
-  pip install pytest pytest-cov pytest-xdist
 }
 
 run_sglang_tests() {
@@ -630,24 +639,34 @@ run_sglang_tests() {
   echo "***************************************************"
 
   run_sglang_install
+  run_test_deps_install
   cd sglang
   run_pytest_command -vvv -n ${PYTEST_MAX_PROCESSES:-4} test/srt/test_triton_attention_kernels.py
 }
 
-run_liger_tests() {
+run_liger_install() {
   echo "************************************************"
-  echo "******    Running Liger Triton tests      ******"
+  echo "******    Installing Liger-Kernel         ******"
   echo "************************************************"
 
   if ! [ -d "./Liger-Kernel" ]; then
     git clone https://github.com/linkedin/Liger-Kernel
   fi
 
   if ! pip list | grep "liger_kernel" ; then
-    pip install pytest pytest-xdist pytest-cov transformers pandas pytest datasets -e Liger-Kernel
+    pip install transformers pandas datasets -e Liger-Kernel
   fi
+}
+
+
+run_liger_tests() {
+  echo "************************************************"
+  echo "******    Running Liger-Kernel tests      ******"
+  echo "************************************************"
 
-  run_pytest_command -vvv -n ${PYTEST_MAX_PROCESSES:-4} Liger-Kernel/test/
+  run_liger_install
+  run_test_deps_install
+  run_pytest_command -vvv Liger-Kernel/test/
 }
 
 run_vllm_install() {
@@ -678,7 +697,7 @@ run_vllm_install() {
     VLLM_TARGET_DEVICE=xpu pip install --no-deps --no-build-isolation -e vllm
   fi
 
-  pip install pytest pytest-cov pytest-xdist cachetools cbor2 blake3 pybase64 openai_harmony tblib
+  pip install cachetools cbor2 blake3 pybase64 openai_harmony tblib
 }
 
 
@@ -688,6 +707,7 @@ run_vllm_tests() {
   echo "************************************************"
 
   run_vllm_install
+  run_test_deps_install
 
   cd vllm
   run_pytest_command -vvv tests/kernels/moe/test_batched_moe.py tests/kernels/attention/test_triton_unified_attention.py
@@ -774,15 +794,18 @@ test_triton() {
   if [ "$TEST_SGLANG" == true ]; then
     run_sglang_tests
   fi
+  if [ "$INSTALL_LIGER" == true ]; then
+    run_liger_install
+  fi
   if [ "$TEST_LIGER" == true ]; then
     run_liger_tests
   fi
-  if [ "$TEST_VLLM" == true ]; then
-    run_vllm_tests
-  fi
   if [ "$INSTALL_VLLM" == true ]; then
     run_vllm_install
   fi
+  if [ "$TEST_VLLM" == true ]; then
+    run_vllm_tests
+  fi
   if [ "$TEST_TRITON_KERNELS" == true ]; then
     run_triton_kernels_tests
   fi